Fix crash with non-ASCII filenames; change CSV to TSV

3 files changed, 85 insertions, 74 deletions

diff --git a/funcs.cpp b/funcs.cpp
index a8e1b25..84cdf96 100644
--- a/funcs.cpp
+++ b/funcs.cpp
@@ -1,4 +1,5 @@
 #include "transcode.h"
+#include <codecvt>
 
 string random_anum_string(int len)
 {
@@ -12,7 +13,7 @@ string random_anum_string(int len)
   return alphan.substr(0, len);
 }
 
-bool to_bool(const wstring& s)
+bool to_bool(const string& s)
 {
   return s != "0";
 }
@@ -23,7 +24,7 @@ void uniq(vector <wstring>& vec)
   vec.erase(unique(vec.begin(), vec.end()), vec.end());
 }
 
-inline wstring& wtrim(wstring& s, const wchar_t* t = L" \t\n\r\f\v")
+inline string& trim(string& s, const char *t = " \t\n\r\f\v")
 {
   s.erase(s.find_last_not_of(t) + 1);
   s.erase(0, s.find_first_not_of(t));
@@ -31,21 +32,21 @@ inline wstring& wtrim(wstring& s, const wchar_t* t = L" \t\n\r\f\v")
 }
 
 void read_config_file(const char* configfile,
-    map<wstring, wstring>& config_options)
+    map<string, string>& config_options)
 {
-  wifstream config(configfile);
+  ifstream config(configfile);
   if (!config.is_open()) return;
 
-  wstring line;
+  string line;
   while(getline(config, line))
   {
-    wistringstream config_line(line);
-    wstring key;
-    if(getline(config_line, key, L'=')) {
-      wtrim(key);
-      wstring value;
+    istringstream config_line(line);
+    string key;
+    if(getline(config_line, key, '=')) {
+      trim(key);
+      string value;
       if(getline(config_line, value)) {
-        wtrim(value);
+        trim(value);
         config_options.insert({key, value});
       }
     }
@@ -55,13 +56,15 @@ void read_config_file(const char* configfile,
 }
 
 void write_config_file(const char* configfile,
-    map<wstring, wstring>& config_options)
+    map<string, string>& config_options)
 {
-  wofstream config(configfile);
+  ofstream config(configfile);
 
   for (auto const& config_option: config_options) {
-    config << config_option.first << " = "
-               << config_option.second << endl;
+    string key = config_option.first;
+    string val = config_option.second;
+
+    config << key << " = " << val << endl;
   }
 
   config.close();
@@ -138,7 +141,7 @@ void get_default_pfont(wstring& default_pfont, const char* styles_xmlfile)
 }
 
 
-void update_default_pfont(const char * styles_xmlfile, wstring& targ_font,
+void update_default_pfont(const char * styles_xmlfile, const wstring& targ_font,
      const vector<wstring>& doc_fonts)
 {
   xml_document styles_xmldoc;
@@ -181,7 +184,7 @@ void detect_fonts(const char* xmlfile, vector<wstring>& doc_fonts,
   uniq(doc_fonts);
 }
 
-void process_xml(const char * xmlfile, wstring& targ_font,
+void process_xml(const char * xmlfile, const wstring& targ_font,
      const vector<wstring>& doc_fonts, const vector<wstring>& doc_encs,
      map<wstring, vector<wstring>>& char_map,
      vector<wstring>& utf_chars, const wstring& default_font, bool do_greek)
diff --git a/transcode.h b/transcode.h
index 204ea0b..6e5e96b 100644
--- a/transcode.h
+++ b/transcode.h
@@ -5,6 +5,7 @@
 #include <map>
 #include <algorithm>
 #include <random>
+#include <locale>
 #include <wx/wx.h>
 #include <wx/stdpaths.h>
 #include <wx/fontdlg.h>
@@ -32,21 +33,23 @@ static string path_separator = "/";
 #ifdef EXTERN_WX_GUI
 extern void detect_fonts(const char* xmlfile, vector<wstring>& doc_fonts,
                               vector<wstring>& doc_encs);
-extern void process_xml(const char * xmlfile, wstring& targ_font,
+extern void process_xml(const char * xmlfile, const wstring& targ_font,
     const vector<wstring>& doc_fonts, const vector<wstring>& doc_encs,
     map<wstring, vector<wstring>>& char_map,
     vector<wstring>& utf_chars, const wstring& default_font, bool do_greek);
-extern bool to_bool(const wstring& s);
+extern bool to_bool(const string& s);
 extern void uniq(vector <wstring>& vec);
 extern void get_default_pfont(wstring& default_font,
     const char* styles_xmlfile);
 extern void update_default_pfont(const char * styles_xmlfile,
-    wstring& targ_font, const vector<wstring>& doc_fonts);
+    const wstring& targ_font, const vector<wstring>& doc_fonts);
 extern string random_anum_string(int len);
 extern void read_config_file(const char* configfile,
-        map<wstring, wstring>& config_options);
+        map<string, string>& config_options);
 extern void write_config_file(const char* configfile,
-        map<wstring, wstring>& config_options);
+        map<string, string>& config_options);
+
+static const char *styles_xmlfile = "word/styles.xml";
 #endif
 
 class TranscodeGUI: public wxApp {
@@ -66,7 +69,7 @@ class TranscodeFrame: public wxFrame {
       bool do_greek;
       bool do_font_select;
     } config_params;
-    map<wstring, wstring> config_options;
+    map<string, string> config_options;
     wxString config_filepath;
 
     wxBoxSizer *infile_sizer;
@@ -76,19 +79,19 @@ class TranscodeFrame: public wxFrame {
     void SelectDoc(wxCommandEvent& event);
     wxString infile_path;
 
-    wxBoxSizer *csvfile_sizer;
-    wxStaticText *csvfile_label;
-    wxTextCtrl *csvfile_field;
-    wxButton *csvfile_button;
-    void SelectCSV(wxCommandEvent& event);
-    wxString csvfile_path;
+    wxBoxSizer *tsvfile_sizer;
+    wxStaticText *tsvfile_label;
+    wxTextCtrl *tsvfile_field;
+    wxButton *tsvfile_button;
+    void SelectTSV(wxCommandEvent& event);
+    wxString tsvfile_path;
 
     wxStaticText *fontsel_label;
     wxBoxSizer *fontsel_sizer;
     wxTextCtrl *fontsel_field;
     wxButton *fontsel_button;
     void DoFontDialog(wxCommandEvent& event);
-    wstring fontsel_name;
+    wxString fontsel_name;
 
     wxCheckBox *check_greek;
     void ChkGrk(wxCommandEvent& event);
@@ -122,8 +125,6 @@ static const vector<const char*> convfiles = {
   "word/footnotes.xml"
 };
 
-static const char * styles_xmlfile = "word/styles.xml";
-
 static const map<const wchar_t, const wchar_t> ascii_utf = {
   {0x2126, 0x03A9} // Ohm symbol
 };
diff --git a/wxgui.cpp b/wxgui.cpp
index 83007de..f199b96 100644
--- a/wxgui.cpp
+++ b/wxgui.cpp
@@ -5,6 +5,12 @@ IMPLEMENT_APP(TranscodeGUI)
 
 bool TranscodeGUI::OnInit()
 {
+#ifdef _WIN32
+  setlocale(LC_ALL, ".UTF8");
+  SetConsoleOutputCP(CP_UTF8);
+#else
+  std::locale::global(std::locale(""));
+#endif
   TranscodeFrame *transcodeFrame = new TranscodeFrame;
   transcodeFrame->Show(true);
 
@@ -26,8 +32,8 @@ TranscodeFrame::TranscodeFrame():
                              + ".slavconvertrc";
 #endif
   read_config_file(config_filepath, config_options);
-  config_params.do_greek = to_bool(config_options[L"greek"]);
-  config_params.do_font_select = to_bool(config_options[L"fontsel"]);
+  config_params.do_greek = to_bool(config_options["greek"]);
+  config_params.do_font_select = to_bool(config_options["fontsel"]);
 
   main_panel = new wxPanel(this, wxID_ANY);
   main_sizer = new wxBoxSizer(wxVERTICAL);
@@ -44,28 +50,28 @@ TranscodeFrame::TranscodeFrame():
   infile_sizer->Add(infile_field, 2, wxEXPAND | wxALL);
   infile_sizer->Add(infile_button, 0, wxALL | wxALIGN_CENTER_VERTICAL);
   main_sizer->Add(infile_sizer, 0, wxEXPAND | wxALL, 2);
-  wxString infile_from_cfg = config_options[L"in"];
+  wxString infile_from_cfg = config_options["in"];
   if (wxFileExists(infile_from_cfg)) {
     infile_path = infile_from_cfg;
     infile_field->ChangeValue(infile_path);
   }
 
-  csvfile_sizer = new wxBoxSizer(wxHORIZONTAL);
-  csvfile_label = new wxStaticText(main_panel, wxID_ANY,
+  tsvfile_sizer = new wxBoxSizer(wxHORIZONTAL);
+  tsvfile_label = new wxStaticText(main_panel, wxID_ANY,
                          wxT("Character Table (TSV)"),
                          wxDefaultPosition, wxDefaultSize);
-  csvfile_field = new wxTextCtrl(main_panel, wxID_ANY, wxEmptyString,
+  tsvfile_field = new wxTextCtrl(main_panel, wxID_ANY, wxEmptyString,
                               wxDefaultPosition, wxDefaultSize,
                               wxCB_READONLY);
-  csvfile_button = new wxButton(main_panel, wxID_ANY, wxT("Select"));
-  main_sizer->Add(csvfile_label, 0, wxALL);
-  csvfile_sizer->Add(csvfile_field, 2, wxEXPAND | wxALL);
-  csvfile_sizer->Add(csvfile_button, 0, wxALL | wxALIGN_CENTER_VERTICAL);
-  main_sizer->Add(csvfile_sizer, 0, wxEXPAND | wxALL, 2);
-  wxString csvfile_from_cfg = config_options[L"csv"];
-  if (wxFileExists(csvfile_from_cfg)) {
-    csvfile_path = csvfile_from_cfg;
-    csvfile_field->ChangeValue(csvfile_path);
+  tsvfile_button = new wxButton(main_panel, wxID_ANY, wxT("Select"));
+  main_sizer->Add(tsvfile_label, 0, wxALL);
+  tsvfile_sizer->Add(tsvfile_field, 2, wxEXPAND | wxALL);
+  tsvfile_sizer->Add(tsvfile_button, 0, wxALL | wxALIGN_CENTER_VERTICAL);
+  main_sizer->Add(tsvfile_sizer, 0, wxEXPAND | wxALL, 2);
+  wxString tsvfile_from_cfg = config_options["tsv"];
+  if (wxFileExists(tsvfile_from_cfg)) {
+    tsvfile_path = tsvfile_from_cfg;
+    tsvfile_field->ChangeValue(tsvfile_path);
   }
 
   fontsel_sizer = new wxBoxSizer(wxHORIZONTAL);
@@ -75,7 +81,7 @@ TranscodeFrame::TranscodeFrame():
   fontsel_field = new wxTextCtrl(main_panel, wxID_ANY, wxEmptyString,
                               wxDefaultPosition, wxDefaultSize,
                               wxCB_READONLY);
-  fontsel_name = config_options[L"font"];
+  fontsel_name = config_options["font"];
   if (fontsel_name.length() > 0)
     fontsel_field->ChangeValue(fontsel_name);
   fontsel_button = new wxButton(main_panel, wxID_ANY, wxT("Select"));
@@ -85,7 +91,7 @@ TranscodeFrame::TranscodeFrame():
   main_sizer->Add(fontsel_sizer, 0, wxEXPAND | wxALL, 2);
 
   infile_button->Bind(wxEVT_BUTTON, &TranscodeFrame::SelectDoc, this);
-  csvfile_button->Bind(wxEVT_BUTTON, &TranscodeFrame::SelectCSV, this);
+  tsvfile_button->Bind(wxEVT_BUTTON, &TranscodeFrame::SelectTSV, this);
   fontsel_button->Bind(wxEVT_BUTTON, &TranscodeFrame::DoFontDialog, this);
 
   check_greek = new wxCheckBox(main_panel, wxID_ANY,
@@ -139,14 +145,14 @@ void TranscodeFrame::Reset(wxCommandEvent& WXUNUSED(event))
   status_messages->Clear();
 
   infile_field->Clear();
-  config_options[L"in"] = L"";
+  config_options["in"] = "";
   infile_path.Clear(); // wxString
-  csvfile_field->Clear();
-  csvfile_path.Clear(); // wxString
-  config_options[L"csv"] = L"";
+  tsvfile_field->Clear();
+  tsvfile_path.Clear(); // wxString
+  config_options["tsv"] = "";
   fontsel_field->Clear();
   fontsel_name.clear(); // wstring
-  config_options[L"font"] = L"";
+  config_options["font"] = "";
 
   write_config_file(config_filepath, config_options);
 }
@@ -161,31 +167,31 @@ void TranscodeFrame::SelectDoc(wxCommandEvent& WXUNUSED(event))
   if (docx_dialog.ShowModal() == wxID_OK) {
     infile_path = docx_dialog.GetPath();
     infile_field->ChangeValue(infile_path);
-    config_options[L"in"] = infile_path;
+    config_options["in"] = infile_path;
     write_config_file(config_filepath, config_options);
   }
-  else if ( !(csvfile_path.length() > 0
-      && !wxFileExists(csvfile_path)) ) {
+  else if ( !(tsvfile_path.length() > 0
+      && !wxFileExists(tsvfile_path)) ) {
     wxMessageBox(wxT("Please select a valid file!"), wxT("Error"),
                         wxOK | wxICON_ERROR);
   }
 }
 
-void TranscodeFrame::SelectCSV(wxCommandEvent& WXUNUSED(event))
+void TranscodeFrame::SelectTSV(wxCommandEvent& WXUNUSED(event))
 {
-  wxFileDialog csv_dialog(this, wxT("Select Character Table File"),
+  wxFileDialog tsv_dialog(this, wxT("Select Character Table File"),
          wxEmptyString, wxEmptyString,
          "Tab-Separated Data (*.tsv)|*.tsv",
          wxFD_OPEN | wxFD_FILE_MUST_EXIST);
 
-  if (csv_dialog.ShowModal() == wxID_OK) {
-    csvfile_path = csv_dialog.GetPath();
-    csvfile_field->ChangeValue(csvfile_path);
-    config_options[L"csv"] = csvfile_path;
+  if (tsv_dialog.ShowModal() == wxID_OK) {
+    tsvfile_path = tsv_dialog.GetPath();
+    tsvfile_field->ChangeValue(tsvfile_path);
+    config_options["tsv"] = tsvfile_path;
     write_config_file(config_filepath, config_options);
   }
-  else if ( !(csvfile_path.length() > 0
-      && wxFileExists(csvfile_path)) ) {
+  else if ( !(tsvfile_path.length() > 0
+      && wxFileExists(tsvfile_path)) ) {
     wxMessageBox(wxT("Please select a valid file!"), wxT("Error"),
                         wxOK | wxICON_ERROR);
   }
@@ -244,7 +250,7 @@ void TranscodeFrame::DoFontDialog(wxCommandEvent& WXUNUSED(event))
     fontsel_name = font_chosen.GetFaceName();
     fontsel_field->ChangeValue(fontsel_name);
   }
-  config_options[L"font"] = fontsel_name;
+  config_options["font"] = fontsel_name;
   write_config_file(config_filepath, config_options);
 }
 
@@ -252,10 +258,10 @@ void TranscodeFrame::ChkGrk(wxCommandEvent& WXUNUSED(event))
 {
   if (check_greek->GetValue()) {
     config_params.do_greek = true;
-    config_options[L"greek"] = L"1";
+    config_options["greek"] = "1";
   } else {
     config_params.do_greek = false;
-    config_options[L"greek"] = L"0";
+    config_options["greek"] = "0";
   }
 
   write_config_file(config_filepath, config_options);
@@ -265,10 +271,10 @@ void TranscodeFrame::ChkFontSel(wxCommandEvent& WXUNUSED(event))
 {
   if (check_fontsel->GetValue()) {
     config_params.do_font_select = true;
-    config_options[L"fontsel"] = L"1";
+    config_options["fontsel"] = "1";
   } else {
     config_params.do_font_select = false;
-    config_options[L"fontsel"] = L"0";
+    config_options["fontsel"] = "0";
   }
 
   write_config_file(config_filepath, config_options);
@@ -335,7 +341,7 @@ void TranscodeFrame::Transcode(wxCommandEvent& WXUNUSED(event))
   cerr.rdbuf(cout.rdbuf());
   wxStreamToTextRedirector redirect(status_messages);
 
-  if ( !(csvfile_path.length() > 0 && infile_path.length() > 0
+  if ( !(tsvfile_path.length() > 0 && infile_path.length() > 0
       && fontsel_name.length() > 0) ) {
     wxMessageBox(wxT("Please select input file, character table, and font!"),
         wxT("Error"), wxOK | wxICON_ERROR);
@@ -357,7 +363,7 @@ void TranscodeFrame::Transcode(wxCommandEvent& WXUNUSED(event))
 
   cout << "Processing file " << infile_path << endl;
 
-  TranscodeFrame::LoadCharmap(csvfile_path, L'\t', char_map, utf_chars);
+  TranscodeFrame::LoadCharmap(tsvfile_path, L'\t', char_map, utf_chars);
 
   wxString tempdir = wxFileName::GetTempDir();
   wxString temp_path = tempdir + path_separator + random_anum_string(6);
@@ -400,12 +406,13 @@ void TranscodeFrame::Transcode(wxCommandEvent& WXUNUSED(event))
   for (const auto& convfile: convfiles) {
     if (!wxFileExists(convfile)) continue;
     cout << "Processing XML source file: " << convfile << endl;
-    process_xml(convfile, fontsel_name, doc_fonts_sel, doc_encs,
+    process_xml(convfile, fontsel_name.ToStdWstring(),
+         doc_fonts_sel, doc_encs,
          char_map, utf_chars, default_paragraph_font,
          config_params.do_greek);
   }
 
-  update_default_pfont(styles_xmlfile, fontsel_name, doc_fonts);
+  update_default_pfont(styles_xmlfile, fontsel_name.ToStdWstring(), doc_fonts);
 
   cout << "Conversion completed!" << endl;