root/lib/charsets.c

/*   */

DEFINITIONS

1. new_codepage_desc
2. free_codepage_desc
3. load_codepages_list_from_file
4. translate_character
5. load_codepages_list
6. free_codepages_list
7. get_codepage_id
8. get_codepage_index
9. is_supported_encoding
10. init_translation_table
11. convert_to_display
12. str_nconvert_to_display
13. convert_from_input
14. str_nconvert_to_input
15. convert_from_utf_to_current
16. convert_from_utf_to_current_c
17. convert_from_8bit_to_utf_c
18. convert_from_8bit_to_utf_c2

   1 /*
   2    Text conversion from one charset to another.
   3 
   4    Copyright (C) 2001-2023
   5    Free Software Foundation, Inc.
   6 
   7    Written by:
   8    Walery Studennikov <despair@sama.ru>
   9 
  10    This file is part of the Midnight Commander.
  11 
  12    The Midnight Commander is free software: you can redistribute it
  13    and/or modify it under the terms of the GNU General Public License as
  14    published by the Free Software Foundation, either version 3 of the License,
  15    or (at your option) any later version.
  16 
  17    The Midnight Commander is distributed in the hope that it will be useful,
  18    but WITHOUT ANY WARRANTY; without even the implied warranty of
  19    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20    GNU General Public License for more details.
  21 
  22    You should have received a copy of the GNU General Public License
  23    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24  */
  25 
  26 /** \file charsets.c
  27  *  \brief Source: Text conversion from one charset to another
  28  */
  29 
  30 #include <config.h>
  31 
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <string.h>
  35 
  36 #include "lib/global.h"
  37 #include "lib/strutil.h"        /* utf-8 functions */
  38 #include "lib/fileloc.h"
  39 #include "lib/util.h"           /* whitespace() */
  40 
  41 #include "lib/charsets.h"
  42 
  43 /*** global variables ****************************************************************************/
  44 
  45 GPtrArray *codepages = NULL;
  46 
  47 unsigned char conv_displ[256];
  48 unsigned char conv_input[256];
  49 
  50 const char *cp_display = NULL;
  51 const char *cp_source = NULL;
  52 
  53 /*** file scope macro definitions ****************************************************************/
  54 
  55 #define UNKNCHAR '\001'
  56 
  57 #define OTHER_8BIT "Other_8_bit"
  58 
  59 /*** file scope type declarations ****************************************************************/
  60 
  61 /*** forward declarations (file scope functions) *************************************************/
  62 
  63 /*** file scope variables ************************************************************************/
  64 
  65 /* --------------------------------------------------------------------------------------------- */
  66 /*** file scope functions ************************************************************************/
  67 /* --------------------------------------------------------------------------------------------- */
  68 
  69 static codepage_desc *
  70 new_codepage_desc (const char *id, const char *name)
     /*   */
  71 {
  72     codepage_desc *desc;
  73 
  74     desc = g_new (codepage_desc, 1);
  75     desc->id = g_strdup (id);
  76     desc->name = g_strdup (name);
  77 
  78     return desc;
  79 }
  80 
  81 /* --------------------------------------------------------------------------------------------- */
  82 
  83 static void
  84 free_codepage_desc (gpointer data)
     /*   */
  85 {
  86     codepage_desc *desc = (codepage_desc *) data;
  87 
  88     g_free (desc->id);
  89     g_free (desc->name);
  90     g_free (desc);
  91 }
  92 
  93 /* --------------------------------------------------------------------------------------------- */
  94 /* returns display codepage */
  95 
  96 static void
  97 load_codepages_list_from_file (GPtrArray ** list, const char *fname)
     /*   */
  98 {
  99     FILE *f;
 100     char buf[BUF_MEDIUM];
 101     char *default_codepage = NULL;
 102 
 103     f = fopen (fname, "r");
 104     if (f == NULL)
 105         return;
 106 
 107     while (fgets (buf, sizeof buf, f) != NULL)
 108     {
 109         /* split string into id and cpname */
 110         char *p = buf;
 111         size_t buflen;
 112 
 113         if (*p == '\n' || *p == '\0' || *p == '#')
 114             continue;
 115 
 116         buflen = strlen (buf);
 117 
 118         if (buflen != 0 && buf[buflen - 1] == '\n')
 119             buf[buflen - 1] = '\0';
 120         while (*p != '\0' && !whitespace (*p))
 121             ++p;
 122         if (*p == '\0')
 123             goto fail;
 124 
 125         *p++ = '\0';
 126         g_strstrip (p);
 127         if (*p == '\0')
 128             goto fail;
 129 
 130         if (strcmp (buf, "default") == 0)
 131             default_codepage = g_strdup (p);
 132         else
 133         {
 134             const char *id = buf;
 135 
 136             if (*list == NULL)
 137             {
 138                 *list = g_ptr_array_sized_new (16);
 139                 g_ptr_array_set_free_func (*list, free_codepage_desc);
 140                 g_ptr_array_add (*list, new_codepage_desc (id, p));
 141             }
 142             else
 143             {
 144                 unsigned int i;
 145 
 146                 /* whether id is already present in list */
 147                 /* if yes, overwrite description */
 148                 for (i = 0; i < (*list)->len; i++)
 149                 {
 150                     codepage_desc *desc;
 151 
 152                     desc = (codepage_desc *) g_ptr_array_index (*list, i);
 153 
 154                     if (strcmp (id, desc->id) == 0)
 155                     {
 156                         /* found */
 157                         g_free (desc->name);
 158                         desc->name = g_strdup (p);
 159                         break;
 160                     }
 161                 }
 162 
 163                 /* not found */
 164                 if (i == (*list)->len)
 165                     g_ptr_array_add (*list, new_codepage_desc (id, p));
 166             }
 167         }
 168     }
 169 
 170     if (default_codepage != NULL)
 171     {
 172         mc_global.display_codepage = get_codepage_index (default_codepage);
 173         g_free (default_codepage);
 174     }
 175 
 176   fail:
 177     fclose (f);
 178 }
 179 
 180 /* --------------------------------------------------------------------------------------------- */
 181 
 182 static char
 183 translate_character (GIConv cd, char c)
     /*   */
 184 {
 185     gchar *tmp_buff = NULL;
 186     gsize bytes_read, bytes_written = 0;
 187     const char *ibuf = &c;
 188     char ch = UNKNCHAR;
 189     int ibuflen = 1;
 190 
 191     tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
 192     if (tmp_buff != NULL)
 193         ch = tmp_buff[0];
 194     g_free (tmp_buff);
 195     return ch;
 196 }
 197 
 198 /* --------------------------------------------------------------------------------------------- */
 199 /*** public functions ****************************************************************************/
 200 /* --------------------------------------------------------------------------------------------- */
 201 
 202 void
 203 load_codepages_list (void)
     /*   */
 204 {
 205     char *fname;
 206 
 207     /* 1: try load /usr/share/mc/mc.charsets */
 208     fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
 209     load_codepages_list_from_file (&codepages, fname);
 210     g_free (fname);
 211 
 212     /* 2: try load /etc/mc/mc.charsets */
 213     fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
 214     load_codepages_list_from_file (&codepages, fname);
 215     g_free (fname);
 216 
 217     if (codepages == NULL)
 218     {
 219         /* files are not found, add default codepage */
 220         fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
 221 
 222         codepages = g_ptr_array_new_with_free_func (free_codepage_desc);
 223         g_ptr_array_add (codepages, new_codepage_desc (DEFAULT_CHARSET, _("7-bit ASCII")));
 224     }
 225 }
 226 
 227 /* --------------------------------------------------------------------------------------------- */
 228 
 229 void
 230 free_codepages_list (void)
     /*   */
 231 {
 232     g_ptr_array_free (codepages, TRUE);
 233     /* NULL-ize pointer to make unit tests happy */
 234     codepages = NULL;
 235 }
 236 
 237 /* --------------------------------------------------------------------------------------------- */
 238 
 239 const char *
 240 get_codepage_id (const int n)
     /*   */
 241 {
 242     return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
 243 }
 244 
 245 /* --------------------------------------------------------------------------------------------- */
 246 
 247 int
 248 get_codepage_index (const char *id)
     /*   */
 249 {
 250     size_t i;
 251 
 252     if (codepages == NULL)
 253         return -1;
 254     if (strcmp (id, OTHER_8BIT) == 0)
 255         return -1;
 256     for (i = 0; i < codepages->len; i++)
 257         if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
 258             return i;
 259     return -1;
 260 }
 261 
 262 /* --------------------------------------------------------------------------------------------- */
 263 /** Check if specified encoding can be used in mc.
 264  * @param encoding name of encoding
 265  * @return TRUE if encoding is supported by mc, FALSE otherwise
 266  */
 267 
 268 gboolean
 269 is_supported_encoding (const char *encoding)
     /*   */
 270 {
 271     gboolean result = FALSE;
 272     guint t;
 273 
 274     for (t = 0; t < codepages->len; t++)
 275     {
 276         const char *id;
 277 
 278         id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
 279         result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
 280     }
 281 
 282     return result;
 283 }
 284 
 285 /* --------------------------------------------------------------------------------------------- */
 286 
 287 char *
 288 init_translation_table (int cpsource, int cpdisplay)
     /*   */
 289 {
 290     int i;
 291     GIConv cd;
 292 
 293     /* Fill inpit <-> display tables */
 294 
 295     if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
 296     {
 297         for (i = 0; i <= 255; ++i)
 298         {
 299             conv_displ[i] = i;
 300             conv_input[i] = i;
 301         }
 302         cp_source = cp_display;
 303         return NULL;
 304     }
 305 
 306     for (i = 0; i <= 127; ++i)
 307     {
 308         conv_displ[i] = i;
 309         conv_input[i] = i;
 310     }
 311     cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
 312     cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
 313 
 314     /* display <- inpit table */
 315 
 316     cd = g_iconv_open (cp_display, cp_source);
 317     if (cd == INVALID_CONV)
 318         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
 319 
 320     for (i = 128; i <= 255; ++i)
 321         conv_displ[i] = translate_character (cd, i);
 322 
 323     g_iconv_close (cd);
 324 
 325     /* inpit <- display table */
 326 
 327     cd = g_iconv_open (cp_source, cp_display);
 328     if (cd == INVALID_CONV)
 329         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
 330 
 331     for (i = 128; i <= 255; ++i)
 332     {
 333         unsigned char ch;
 334         ch = translate_character (cd, i);
 335         conv_input[i] = (ch == UNKNCHAR) ? i : ch;
 336     }
 337 
 338     g_iconv_close (cd);
 339 
 340     return NULL;
 341 }
 342 
 343 /* --------------------------------------------------------------------------------------------- */
 344 
 345 void
 346 convert_to_display (char *str)
     /*   */
 347 {
 348     if (str != NULL)
 349         for (; *str != '\0'; str++)
 350             *str = conv_displ[(unsigned char) *str];
 351 }
 352 
 353 /* --------------------------------------------------------------------------------------------- */
 354 
 355 GString *
 356 str_nconvert_to_display (const char *str, int len)
     /*   */
 357 {
 358     GString *buff;
 359     GIConv conv;
 360 
 361     if (str == NULL)
 362         return NULL;
 363 
 364     if (cp_display == cp_source)
 365         return g_string_new (str);
 366 
 367     conv = str_crt_conv_from (cp_source);
 368 
 369     buff = g_string_new ("");
 370     str_nconvert (conv, str, len, buff);
 371     str_close_conv (conv);
 372     return buff;
 373 }
 374 
 375 /* --------------------------------------------------------------------------------------------- */
 376 
 377 void
 378 convert_from_input (char *str)
     /*   */
 379 {
 380     if (str != NULL)
 381         for (; *str != '\0'; str++)
 382             *str = conv_input[(unsigned char) *str];
 383 }
 384 
 385 /* --------------------------------------------------------------------------------------------- */
 386 
 387 GString *
 388 str_nconvert_to_input (const char *str, int len)
     /*   */
 389 {
 390     GString *buff;
 391     GIConv conv;
 392 
 393     if (str == NULL)
 394         return NULL;
 395 
 396     if (cp_display == cp_source)
 397         return g_string_new (str);
 398 
 399     conv = str_crt_conv_to (cp_source);
 400 
 401     buff = g_string_new ("");
 402     str_nconvert (conv, str, len, buff);
 403     str_close_conv (conv);
 404     return buff;
 405 }
 406 
 407 /* --------------------------------------------------------------------------------------------- */
 408 
 409 unsigned char
 410 convert_from_utf_to_current (const char *str)
     /*   */
 411 {
 412     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 413     unsigned char ch = '.';
 414     GIConv conv;
 415     const char *cp_to;
 416 
 417     if (str == NULL)
 418         return '.';
 419 
 420     cp_to = get_codepage_id (mc_global.source_codepage);
 421     conv = str_crt_conv_to (cp_to);
 422 
 423     if (conv != INVALID_CONV)
 424     {
 425         switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
 426         {
 427         case ESTR_SUCCESS:
 428             ch = buf_ch[0];
 429             break;
 430         case ESTR_PROBLEM:
 431         case ESTR_FAILURE:
 432             ch = '.';
 433             break;
 434         default:
 435             break;
 436         }
 437         str_close_conv (conv);
 438     }
 439 
 440     return ch;
 441 }
 442 
 443 /* --------------------------------------------------------------------------------------------- */
 444 
 445 unsigned char
 446 convert_from_utf_to_current_c (int input_char, GIConv conv)
     /*   */
 447 {
 448     unsigned char str[UTF8_CHAR_LEN + 1];
 449     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 450     unsigned char ch = '.';
 451     int res;
 452 
 453     res = g_unichar_to_utf8 (input_char, (char *) str);
 454     if (res == 0)
 455         return ch;
 456 
 457     str[res] = '\0';
 458 
 459     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 460     {
 461     case ESTR_SUCCESS:
 462         ch = buf_ch[0];
 463         break;
 464     case ESTR_PROBLEM:
 465     case ESTR_FAILURE:
 466         ch = '.';
 467         break;
 468     default:
 469         break;
 470     }
 471 
 472     return ch;
 473 }
 474 
 475 /* --------------------------------------------------------------------------------------------- */
 476 
 477 int
 478 convert_from_8bit_to_utf_c (char input_char, GIConv conv)
     /*   */
 479 {
 480     unsigned char str[2];
 481     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 482     int ch;
 483 
 484     str[0] = (unsigned char) input_char;
 485     str[1] = '\0';
 486 
 487     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 488     {
 489     case ESTR_SUCCESS:
 490         {
 491             int res;
 492 
 493             res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 494             ch = res >= 0 ? res : buf_ch[0];
 495             break;
 496         }
 497     case ESTR_PROBLEM:
 498     case ESTR_FAILURE:
 499     default:
 500         ch = '.';
 501         break;
 502     }
 503 
 504     return ch;
 505 }
 506 
 507 /* --------------------------------------------------------------------------------------------- */
 508 
 509 int
 510 convert_from_8bit_to_utf_c2 (char input_char)
     /*   */
 511 {
 512     int ch = '.';
 513     GIConv conv;
 514     const char *cp_from;
 515 
 516     cp_from = get_codepage_id (mc_global.source_codepage);
 517 
 518     conv = str_crt_conv_to (cp_from);
 519     if (conv != INVALID_CONV)
 520     {
 521         ch = convert_from_8bit_to_utf_c (input_char, conv);
 522         str_close_conv (conv);
 523     }
 524 
 525     return ch;
 526 }
 527 
 528 /* --------------------------------------------------------------------------------------------- */

/*   */