thodg/libiconv/lib/iconv_open1.h

Branch - branch -master- tag -v1.9.2v1.9.1v1.9v1.8v1.7v1.6.1v1.6v1.5.1v1.5v1.4v1.3v1.2v1.18v1.17v1.16v1.15v1.14v1.13.1v1.13v1.12v1.11.1v1.11v1.10v1.1v1.0v0.3libcharset-1.1clisp-2.28-2002-03-03clisp-2.27-2001-07-17clisp-2.26-2001-05-23clisp-2.25.1-2001-04-061.11.x-branchpoint

• Show log

  Commit

• Hash : 9f28054d
  Author :
  Date : 2024-12-15T12:52:18
  

  Implement the //NON_IDENTICAL_DISCARD suffix from POSIX:2024.
  
  * include/iconv.h.in (ICONV_GET_DISCARD_INVALID,
  ICONV_SET_DISCARD_INVALID, ICONV_GET_DISCARD_NON_IDENTICAL,
  ICONV_SET_DISCARD_NON_IDENTICAL): New macros.
  * lib/converters.h (struct conv_struct): Change type of discard_ilseq to
  'unsigned int'.
  (DISCARD_INVALID, DISCARD_UNCONVERTIBLE): New macros.
  * lib/iconv.c (iconv_open, iconv_open_into): Change type of
  discard_ilseq to 'unsigned int'.
  (iconvctl): Implement ICONV_GET_DISCARD_INVALID,
  ICONV_SET_DISCARD_INVALID, ICONV_GET_DISCARD_NON_IDENTICAL,
  ICONV_SET_DISCARD_NON_IDENTICAL. Change the implementation of
  ICONV_GET_DISCARD_ILSEQ, ICONV_SET_DISCARD_ILSEQ to test/set both bits.
  * lib/iconv_open1.h: Update comment. Recognize //NON_IDENTICAL_DISCARD.
  * lib/iconv_open2.h: Update comment.
  * lib/loop_unicode.h (mb_to_uc_write_replacement): Test the
  DISCARD_UNCONVERTIBLE bit of discard_ilseq.
  (unicode_loop_convert): Test the respective bit of discard_ilseq.
  (unicode_loop_reset): Test the DISCARD_UNCONVERTIBLE bit of
  discard_ilseq.
  * lib/loop_wchar.h (wchar_from_loop_convert, wchar_to_loop_convert):
  Test the DISCARD_INVALID bit of discard_ilseq.
  * man/iconv_open.3: Mention the //NON_IDENTICAL_DISCARD suffix. Mark as
  conforming to POSIX:2024.
  * man/iconv.3: Likewise.
  * man/iconv_close.3: Mark as conforming to POSIX:2024.
  * man/iconv.1: Likewise.
  * man/iconvctl.3: Document ICONV_GET_DISCARD_INVALID,
  ICONV_SET_DISCARD_INVALID, ICONV_GET_DISCARD_NON_IDENTICAL,
  ICONV_SET_DISCARD_NON_IDENTICAL. Revise the description of
  ICONV_GET_DISCARD_ILSEQ, ICONV_SET_DISCARD_ILSEQ.
  * tests/test-discard.c (test_default, test_translit, test_ignore,
  test_ignore_translit): Test also the ICONV_GET_DISCARD_INVALID,
  ICONV_GET_DISCARD_NON_IDENTICAL accessors.
  (test_nid, test_nid_translit, test_invd, test_invd_translit): New
  functions.
  (main): Add test cases with //NON_IDENTICAL_DISCARD suffix.
  * NEWS: Mention the change.
  

Download

• lib/iconv_open1.h

• /*
   * Copyright (C) 1999-2024 Free Software Foundation, Inc.
   * This file is part of the GNU LIBICONV Library.
   *
   * The GNU LIBICONV Library is free software; you can redistribute it
   * and/or modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either version 2.1
   * of the License, or (at your option) any later version.
   *
   * The GNU LIBICONV Library is distributed in the hope that it will be
   * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with the GNU LIBICONV Library; see the file COPYING.LIB.
   * If not, see <https://www.gnu.org/licenses/>.
   */
  
  /* Part 1 of iconv_open.
     Input: const char* tocode, const char* fromcode.
     Output:
       unsigned int from_index;
       int from_wchar;
       unsigned int from_surface;
       unsigned int to_index;
       int to_wchar;
       unsigned int to_surface;
       int transliterate;
       unsigned int discard_ilseq;
     Jumps to 'invalid' in case of error.
   */
  {
    char buf[MAX_WORD_LENGTH+9+9+1];
    const char* cp;
    char* bp;
    const struct alias * ap;
    unsigned int count;
  
    from_surface = ICONV_SURFACE_NONE;
    to_surface = ICONV_SURFACE_NONE;
    transliterate = 0;
    discard_ilseq = 0;
  
    /* Before calling aliases_lookup, convert the input string to upper case,
     * and check whether it's entirely ASCII (we call gperf with option "-7"
     * to achieve a smaller table) and non-empty. If it's not entirely ASCII,
     * or if it's too long, it is not a valid encoding name.
     */
    for (to_wchar = 0;;) {
      /* Search tocode in the table. */
      for (cp = tocode, bp = buf, count = MAX_WORD_LENGTH+9+9+1; ; cp++, bp++) {
        unsigned char c = (unsigned char) *cp;
        if (c >= 0x80)
          goto invalid;
        if (c >= 'a' && c <= 'z')
          c -= 'a'-'A';
        *bp = c;
        if (c == '\0')
          break;
        if (--count == 0)
          goto invalid;
      }
      for (;;) {
        char *sp = bp;
        int parsed_translit = 0;
        int parsed_ignore = 0;
        int parsed_non_identical_discard = 0;
        for (;;) {
          if (sp-buf > 9 && memcmp(sp-9,"/TRANSLIT",9)==0) {
            sp = sp - 9;
            parsed_translit = 1;
          } else if (sp-buf > 7 && memcmp(sp-7,"/IGNORE",7)==0) {
            sp = sp - 7;
            parsed_ignore = 1;
          } else if (sp-buf > 22 && memcmp(sp-22,"/NON_IDENTICAL_DISCARD",22)==0) {
            sp = sp - 22;
            parsed_non_identical_discard = 1;
          } else
            break;
          if (sp > buf && sp[-1] == '/')
            sp = sp - 1;
        }
        if (sp > buf && sp[-1] == '/') {
          bp = sp - 1;
        } else if (sp-buf >= 9 && memcmp(sp-9,"/ZOS_UNIX",9)==0) {
          bp = sp - 9;
          to_surface = ICONV_SURFACE_EBCDIC_ZOS_UNIX;
        } else
          bp = sp;
        *bp = '\0';
        if (parsed_translit)
          transliterate = 1;
        if (parsed_ignore)
          discard_ilseq |= DISCARD_INVALID | DISCARD_UNCONVERTIBLE;
        if (parsed_non_identical_discard)
          discard_ilseq |= DISCARD_UNCONVERTIBLE;
        break;
      }
      if (buf[0] == '\0') {
        tocode = locale_charset();
        /* Avoid an endless loop that could occur when using an older version
           of localcharset.c. */
        if (tocode[0] == '\0')
          goto invalid;
        continue;
      }
      ap = aliases_lookup(buf,bp-buf);
      if (ap == NULL) {
        ap = aliases2_lookup(buf);
        if (ap == NULL)
          goto invalid;
      }
      if (ap->encoding_index == ei_local_char) {
        tocode = locale_charset();
        /* Avoid an endless loop that could occur when using an older version
           of localcharset.c. */
        if (tocode[0] == '\0')
          goto invalid;
        continue;
      }
      if (ap->encoding_index == ei_local_wchar_t) {
        /* On systems which define __STDC_ISO_10646__, wchar_t is Unicode.
           This is also the case on native Woe32 systems and Cygwin >= 1.7, where
           we know that it is UTF-16.  */
  #if (defined _WIN32 && !defined __CYGWIN__) || (defined __CYGWIN__ && CYGWIN_VERSION_DLL_MAJOR >= 1007)
        if (sizeof(wchar_t) == 4) {
          to_index = ei_ucs4internal;
          break;
        }
        if (sizeof(wchar_t) == 2) {
  # if WORDS_LITTLEENDIAN
          to_index = ei_utf16le;
  # else
          to_index = ei_utf16be;
  # endif
          break;
        }
  #elif __STDC_ISO_10646__
        if (sizeof(wchar_t) == 4) {
          to_index = ei_ucs4internal;
          break;
        }
        if (sizeof(wchar_t) == 2) {
          to_index = ei_ucs2internal;
          break;
        }
        if (sizeof(wchar_t) == 1) {
          to_index = ei_iso8859_1;
          break;
        }
  #endif
  #if HAVE_MBRTOWC
        to_wchar = 1;
        tocode = locale_charset();
        continue;
  #endif
        goto invalid;
      }
      to_index = ap->encoding_index;
      break;
    }
    for (from_wchar = 0;;) {
      /* Search fromcode in the table. */
      for (cp = fromcode, bp = buf, count = MAX_WORD_LENGTH+9+9+1; ; cp++, bp++) {
        unsigned char c = (unsigned char) *cp;
        if (c >= 0x80)
          goto invalid;
        if (c >= 'a' && c <= 'z')
          c -= 'a'-'A';
        *bp = c;
        if (c == '\0')
          break;
        if (--count == 0)
          goto invalid;
      }
      for (;;) {
        char *sp = bp;
        int parsed_translit = 0;
        int parsed_ignore = 0;
        int parsed_non_identical_discard = 0;
        for (;;) {
          if (sp-buf > 9 && memcmp(sp-9,"/TRANSLIT",9)==0) {
            sp = sp - 9;
            parsed_translit = 1;
          } else if (sp-buf > 7 && memcmp(sp-7,"/IGNORE",7)==0) {
            sp = sp - 7;
            parsed_ignore = 1;
          } else if (sp-buf > 22 && memcmp(sp-22,"/NON_IDENTICAL_DISCARD",22)==0) {
            sp = sp - 22;
            parsed_non_identical_discard = 1;
          } else
            break;
          if (sp > buf && sp[-1] == '/')
            sp = sp - 1;
        }
        if (sp > buf && sp[-1] == '/') {
          bp = sp - 1;
        } else if (sp-buf >= 9 && memcmp(sp-9,"/ZOS_UNIX",9)==0) {
          bp = sp - 9;
          from_surface = ICONV_SURFACE_EBCDIC_ZOS_UNIX;
        } else
          bp = sp;
        *bp = '\0';
        if (parsed_translit)
          transliterate = 1;
        if (parsed_ignore)
          discard_ilseq |= DISCARD_INVALID | DISCARD_UNCONVERTIBLE;
        if (parsed_non_identical_discard)
          discard_ilseq |= DISCARD_UNCONVERTIBLE;
        break;
      }
      if (buf[0] == '\0') {
        fromcode = locale_charset();
        /* Avoid an endless loop that could occur when using an older version
           of localcharset.c. */
        if (fromcode[0] == '\0')
          goto invalid;
        continue;
      }
      ap = aliases_lookup(buf,bp-buf);
      if (ap == NULL) {
        ap = aliases2_lookup(buf);
        if (ap == NULL)
          goto invalid;
      }
      if (ap->encoding_index == ei_local_char) {
        fromcode = locale_charset();
        /* Avoid an endless loop that could occur when using an older version
           of localcharset.c. */
        if (fromcode[0] == '\0')
          goto invalid;
        continue;
      }
      if (ap->encoding_index == ei_local_wchar_t) {
        /* On systems which define __STDC_ISO_10646__, wchar_t is Unicode.
           This is also the case on native Woe32 systems and Cygwin >= 1.7, where
           we know that it is UTF-16.  */
  #if (defined _WIN32 && !defined __CYGWIN__) || (defined __CYGWIN__ && CYGWIN_VERSION_DLL_MAJOR >= 1007)
        if (sizeof(wchar_t) == 4) {
          from_index = ei_ucs4internal;
          break;
        }
        if (sizeof(wchar_t) == 2) {
  # if WORDS_LITTLEENDIAN
          from_index = ei_utf16le;
  # else
          from_index = ei_utf16be;
  # endif
          break;
        }
  #elif __STDC_ISO_10646__
        if (sizeof(wchar_t) == 4) {
          from_index = ei_ucs4internal;
          break;
        }
        if (sizeof(wchar_t) == 2) {
          from_index = ei_ucs2internal;
          break;
        }
        if (sizeof(wchar_t) == 1) {
          from_index = ei_iso8859_1;
          break;
        }
  #endif
  #if HAVE_WCRTOMB
        from_wchar = 1;
        fromcode = locale_charset();
        continue;
  #endif
        goto invalid;
      }
      from_index = ap->encoding_index;
      break;
    }
  }
  

Download