1. Important Notice: ALWAYS login to your account using Board4all.biz - we do not own or run any other websites so please make sure you are connected to our official domain, and that you see the secure connection padlock before entering your login details.

How to remove BOM from Text File created in Windows?

Discussion in 'HelpDesk' started by HP Owner, May 16, 2018.

  1. HP

    HP Owner Registered User

    Joined:
    Aug 28, 2017
    Messages:
    130
    Likes Received:
    973
    I have created one Text File using TStreamWriter in Windows environment using RAD Studio 10.2.3. I have used UTF8 Encoding. I have used the following commands to to add lines to the text file
    Code:
    procedure TForm1.Button1Click(Sender: TObject);
    var
      FileName: string;
      StreamWriter: TStreamWriter;
      I, J: integer;
      S1,S2: string;
    begin
      FileName := 'UNIX Test File.txt';
      S1 := 'StartingText ';
      S2 := ' TrailingText';
      StreamWriter := TStreamWriter.Create(FileName, True, TEncoding.UTF8);
      try
        StreamWriter.NewLine := #$A;
        if (((Trim(Edit1.Text) <> '') and (Trim(Memo1.Text) <> ''))) then
          begin
            for I := 0 to Memo1.Lines.Count-1 do
              begin
                StreamWriter.WriteLine(S1 + Memo1.Lines[I] + S2);
                Memo3.Lines.Add(S1 + Memo1.Lines[I] + S2);
              end;
          end;
    
        if (((Trim(Edit2.Text) <> '') and (Trim(Memo2.Text) <> ''))) then
          begin
            for J := 0 to Memo2.Lines.Count-1 do
              begin
                StreamWriter.WriteLine(S1 + Memo2.Lines[J] + S2);
                Memo3.Lines.Add(S1 + Memo2.Lines[J] + S2);
              end;
          end;
      finally
        StreamWriter.Free();
      end;
    end;
    
    I need to run the file in UNIX environment. But the problem is that this file is not UNIX compatible because it has BOM(HEX Value EFBBBF) .i.e Byte Order Mark.

    I have tried to create one blank file modifying my code and found that if I use TEncoding.UTF8 for file creation then BOM is added and if I don't use any Encoding for file creation then BOM is not added. But BOM is added after Memo reading and adding line to the file. So I need remove the BOM.

    How to remove the BOM?
     
    Last edited: May 16, 2018
  2. Skydevil

    Skydevil is a Trusted Warez PosterSkydevil Administrator Staff Member Administrator V.I.P DEV Guild

    Joined:
    Oct 4, 2008
    Messages:
    12,086
    Likes Received:
    25,750
    Define your own encoding...
    Code:
    type
      TUTF8EncodingWithoutPreamble = class(TUTF8Encoding)
      public
        function GetPreamble: System.TArray<System.Byte>; override;
      end;
    
    { TUTF8EncodingWithoutPreamble }
    
    function TUTF8EncodingWithoutPreamble.GetPreamble: System.TArray<System.Byte>;
    begin
      Result := [];
    end;
    
    Please keep in mind, you can create this encoding once and use it anywhere you need, but must free it after use to avoid a memory leak. The class TStreamWriter won't do that for you.
     
    HP Owner likes this.
  3. HP

    HP Owner Registered User

    Joined:
    Aug 28, 2017
    Messages:
    130
    Likes Received:
    973
    Please provide one sample code for better understanding. It will be better for me if you modify my above code. Please! Please!
     
  4. Skydevil

    Skydevil is a Trusted Warez PosterSkydevil Administrator Staff Member Administrator V.I.P DEV Guild

    Joined:
    Oct 4, 2008
    Messages:
    12,086
    Likes Received:
    25,750
    You could combine all related things into one class...
    Code:
    unit System.IO.UnixTextFile;
    
    interface
    
    uses
      System.SysUtils,
      System.Classes,
      System.Math,
      System.IOUtils;
    
    type
      TUnixTextFile = class
      private
        FEncoding: TEncoding;
        FWriter: TStreamWriter;
      public
        constructor Create(const FileName: string; const Append: Boolean);
        destructor Destroy; override;
        class function HasBOM(const FileName: string): Boolean;
        class function CleanBOM(const FileName: string): Boolean;
      public
        procedure Write(const Value: string);
        procedure WriteLine(const Value: string);
      end;
    
    implementation
    
    type
      TUTF8EncodingWithoutPreamble = class(TUTF8Encoding)
      public
        function GetPreamble: System.TArray<System.Byte>; override;
        function Clone: TEncoding; override;
      end;
    
    { TUTF8EncodingWithoutPreamble }
    
    function TUTF8EncodingWithoutPreamble.Clone: TEncoding;
    begin
      Result := TUTF8EncodingWithoutPreamble.Create;
    end;
    
    function TUTF8EncodingWithoutPreamble.GetPreamble: System.TArray<System.Byte>;
    begin
      Result := [];
    end;
    
    const
      MAX_PREAMBLE_SIZE = 4;
    
    { TUnixTextFile }
    
    class function TUnixTextFile.CleanBOM(const FileName: string): Boolean;
    var
      FS: TFileStream;
      Buffer: TArray<Byte>;
      Encoding: TEncoding;
      PreambleLength: Integer;
    begin
      FS := TFileStream.Create(FileName, fmOpenReadWrite, fmShareExclusive);
      try
        SetLength(Buffer, MAX_PREAMBLE_SIZE);
        FS.ReadBuffer(Buffer, System.Math.Min(FS.Size, MAX_PREAMBLE_SIZE));
        Encoding := nil;
        PreambleLength := TEncoding.GetBufferEncoding(Buffer, Encoding);
        Result := PreambleLength > 0;
        if (Result) then
        begin
          SetLength(Buffer, FS.Size - PreambleLength);
          FS.Position := PreambleLength;
          FS.ReadBuffer(Buffer, Length(Buffer));
          FS.Position := 0;
          FS.WriteBuffer(Buffer, Length(Buffer));
          FS.Size := FS.Position;
        end;
      finally
        FS.Free();
      end;
    end;
    
    constructor TUnixTextFile.Create(const FileName: string; const Append: Boolean);
    begin
      inherited Create();
      FEncoding := TUTF8EncodingWithoutPreamble.Create();
      FWriter := TStreamWriter.Create(FileName, Append, FEncoding);
      FWriter.NewLine := #10;   // Unix-conform line style
    end;
    
    destructor TUnixTextFile.Destroy;
    begin
      FWriter.Free();
      FEncoding.Free();
      inherited;
    end;
    
    class function TUnixTextFile.HasBOM(const FileName: string): Boolean;
    var
      FS: TFileStream;
      Buffer: TArray<Byte>;
      Encoding: TEncoding;
      PreambleLength: Integer;
    begin
      FS := TFileStream.Create(FileName, fmOpenRead);
      try
        SetLength(Buffer, MAX_PREAMBLE_SIZE);
        FS.ReadBuffer(Buffer, System.Math.Min(FS.Size, MAX_PREAMBLE_SIZE));
        Encoding := nil;
        PreambleLength := TEncoding.GetBufferEncoding(Buffer, Encoding);
        Result := PreambleLength > 0;
      finally
        FS.Free();
      end;
    end;
    
    procedure TUnixTextFile.Write(const Value: string);
    begin
      FWriter.Write(Value);
    end;
    
    procedure TUnixTextFile.WriteLine(const Value: string);
    begin
      FWriter.WriteLine(Value);
    end;
    
    end.
    
    The using is simple...
    Code:
    procedure TForm1.Button2Click(Sender: TObject);
    var
      TextFile: TUnixTextFile;
    begin
      TextFile := TUnixTextFile.Create('Sample.txt', True);
      try
        TextFile.Write('The');
        TextFile.Write(' ');
        TextFile.Write('quick');
        TextFile.Write(' ');
        TextFile.Write('fox');
        TextFile.Write(' ');
        TextFile.WriteLine('...');
      finally
        TextFile.Free();
      end;
    end;
    
    The result would be something like this...
    Code:
    54 68 65 20 71 75 69 63 6B 20 66 6F 78 20 2E 2E 2E 0A
    
    The class TUnixTextFile has also two static helper functions which could be used in this way...
    Code:
    procedure TForm1.Button3Click(Sender: TObject);
    begin
      if (TUnixTextFile.HasBOM('Sample.txt')) then
      begin
        // Do anything
      end;
    
      if (TUnixTextFile.CleanBOM('SampleBOM.txt')) then
      begin
        // CleanBOM returns True in case the BOM was removed
      end;
    end;
    
     
    ontryit, apol_guerra and HP Owner like this.
  5. HP

    HP Owner Registered User

    Joined:
    Aug 28, 2017
    Messages:
    130
    Likes Received:
    973
    Your code works just mind blowing. Could you do me a favor? I do not like to use UnixTextFile.pas as a separate unit. So I have tried add your code into my Unit1.pas, but unable to get success. I have tried a lot yet unable to compile. I don't like to show my erroneous code. Please! Please!
     
  6. Skydevil

    Skydevil is a Trusted Warez PosterSkydevil Administrator Staff Member Administrator V.I.P DEV Guild

    Joined:
    Oct 4, 2008
    Messages:
    12,086
    Likes Received:
    25,750
    I cannot really understand why the class could not be in separate unit. It would be just easy to reuse it everywhere you need it. But anyway.

    Take care that these uses are already in the uses of the unit you want to insert the class below...
    Code:
      System.SysUtils,
      System.Classes,
      System.Math,
      System.IOUtils;
    
    Add these lines into after "implementation" into the unit where you want to use the class...
    Code:
    type
      TUnixTextFile = class
      private
        FEncoding: TEncoding;
        FWriter: TStreamWriter;
      public
        constructor Create(const FileName: string; const Append: Boolean);
        destructor Destroy; override;
        class function HasBOM(const FileName: string): Boolean;
        class function CleanBOM(const FileName: string): Boolean;
      public
        procedure Write(const Value: string);
        procedure WriteLine(const Value: string);
      end;
    
    type
      TUTF8EncodingWithoutPreamble = class(TUTF8Encoding)
      public
        function GetPreamble: System.TArray<System.Byte>; override;
        function Clone: TEncoding; override;
      end;
    
    { TUTF8EncodingWithoutPreamble }
    
    function TUTF8EncodingWithoutPreamble.Clone: TEncoding;
    begin
      Result := TUTF8EncodingWithoutPreamble.Create;
    end;
    
    function TUTF8EncodingWithoutPreamble.GetPreamble: System.TArray<System.Byte>;
    begin
      Result := [];
    end;
    
    const
      MAX_PREAMBLE_SIZE = 4;
    
    { TUnixTextFile }
    
    class function TUnixTextFile.CleanBOM(const FileName: string): Boolean;
    var
      FS: TFileStream;
      Buffer: TArray<Byte>;
      Encoding: TEncoding;
      PreambleLength: Integer;
    begin
      FS := TFileStream.Create(FileName, fmOpenReadWrite, fmShareExclusive);
      try
        SetLength(Buffer, MAX_PREAMBLE_SIZE);
        FS.ReadBuffer(Buffer, System.Math.Min(FS.Size, MAX_PREAMBLE_SIZE));
        Encoding := nil;
        PreambleLength := TEncoding.GetBufferEncoding(Buffer, Encoding);
        Result := PreambleLength > 0;
        if (Result) then
        begin
          SetLength(Buffer, FS.Size - PreambleLength);
          FS.Position := PreambleLength;
          FS.ReadBuffer(Buffer, Length(Buffer));
          FS.Position := 0;
          FS.WriteBuffer(Buffer, Length(Buffer));
          FS.Size := FS.Position;
        end;
      finally
        FS.Free();
      end;
    end;
    
    constructor TUnixTextFile.Create(const FileName: string; const Append: Boolean);
    begin
      inherited Create();
      FEncoding := TUTF8EncodingWithoutPreamble.Create();
      FWriter := TStreamWriter.Create(FileName, Append, FEncoding);
      FWriter.NewLine := #10;   // Unix-conform line style
    end;
    
    destructor TUnixTextFile.Destroy;
    begin
      FWriter.Free();
      FEncoding.Free();
      inherited;
    end;
    
    class function TUnixTextFile.HasBOM(const FileName: string): Boolean;
    var
      FS: TFileStream;
      Buffer: TArray<Byte>;
      Encoding: TEncoding;
      PreambleLength: Integer;
    begin
      FS := TFileStream.Create(FileName, fmOpenRead);
      try
        SetLength(Buffer, MAX_PREAMBLE_SIZE);
        FS.ReadBuffer(Buffer, System.Math.Min(FS.Size, MAX_PREAMBLE_SIZE));
        Encoding := nil;
        PreambleLength := TEncoding.GetBufferEncoding(Buffer, Encoding);
        Result := PreambleLength > 0;
      finally
        FS.Free();
      end;
    end;
    
    procedure TUnixTextFile.Write(const Value: string);
    begin
      FWriter.Write(Value);
    end;
    
    procedure TUnixTextFile.WriteLine(const Value: string);
    begin
      FWriter.WriteLine(Value);
    end;
    
    Here is an example where to place...
    Code:
    (...)
    var
      Form1: TForm1;
    
    implementation
    
    {$R *.dfm}
    
    uses
      //System.IO.UnixTextFile,
      Unit2;
    
    << INSERT ABOVE LINES HERE >>
    
    procedure TForm1.Button1Click(Sender: TObject);
    begin
      DefaultLogger().LogMethod('Button1Click', Self);
    
      Form2.Show;
    end;
    

    In case you won't get it work. Send me a private message with the code.
     
    ontryit and HP Owner like this.
  7. yu

    yukio Registered User

    Joined:
    Nov 29, 2015
    Messages:
    11
    Likes Received:
    43
    hi. i use TStringList like below.

    procedure TForm1.Button1Click(Sender: TObject);
    var
    str_list:TStringList;
    begin
    str_list:=TStringList.Create;
    str_list.LineBreak:=#10;
    str_list.Text:='text_data';
    str_list.WriteBOM:=false;
    str_list.SaveToFile(test.txt',TEncoding.UTF8);
    str_list.Free;
    end;
     
    ontryit and HP Owner like this.
  8. Pe

    PeterS Registered User

    Joined:
    Dec 24, 2010
    Messages:
    155
    Likes Received:
    1,248
    var
    f: TFileStream;
    begin
    f := TFileStream.Create('test.txt',fmcreate);
    Memo1.Lines.WriteBOM := False;
    Memo1.Lines.SaveToStream(f, TEncoding.UTF8);
    f.Free;
    end;
     
    ontryit and HP Owner like this.

Share This Page