Description: Remove incorrect strcoll implementation.

Author: Carlos O'Donell <codonell@redhat.com>

Origin: git://sourceware.org/glibc.git

Bug-RHEL: N/A

Bug-Fedora: #1276711

Bug-Upstream: #18589

Upstream status: committed

commit 60cf80f09d029257caedc0c8abe7e3e09c64e6c7

Author: Martin Sebor <msebor@gcc.gnu.org>

Date:   Mon Sep 28 16:55:57 2015 -0400

    Let 'make check subdirs=string' succeed even when it's invoked

    immediately after glibc has been built and before 'make check'

    (or after 'make clean').

commit facdd9ea29ab94aac2b188ec3cc41f8733d769e0

Author: Carlos O'Donell <carlos@systemhalted.org>

Date:   Fri Oct 9 16:35:53 2015 -0400

    Fix typo in bug-strcoll2 (Bug 18589)

    Fix the copyright year and remove contributed by in the

    bug-strcoll2 test. In addition add the correct dependency

    on $(gen-locales) to ensure all the test locales are generated.

commit 02018629a1397d03eccceacaf2ee1c50e3c4001c

Author: Carlos O'Donell <carlos@systemhalted.org>

Date:   Thu Oct 8 16:54:30 2015 -0400

    strcoll: Add bug-strcoll2 to testsuite (Bug 18589).

    Adds bug-strcoll2 to the string tests, along with the

    generation of required locales.

commit 87701a58e291bd7ac3b407d10a829dac52c9c16e

Author: Carlos O'Donell <carlos@systemhalted.org>

Date:   Thu Oct 8 16:34:53 2015 -0400

    strcoll: Remove incorrect STRDIFF-based optimization (Bug 18589).

    The optimization introduced in commit

    f13c2a8dff2329c6692a80176262ceaaf8a6f74e, causes regressions in

    sorting for languages that have digraphs that change sort order, like

    cs_CZ which sorts ch between h and i.

    My analysis shows the fast-forwarding optimization in STRCOLL advances

    through a digraph while possibly stopping in the middle which results

    in a subsequent skipping of the digraph and incorrect sorting. The

    optimization is incorrect as implemented and because of that I'm

    removing it for 2.23, and I will also commit this fix for 2.22 where

    it was originally introduced.

    This patch reverts the optimization, introduces a new bug-strcoll2.c

    regression test that tests both cs_CZ.UTF-8 and da_DK.ISO-8859-1 and

    ensures they sort one digraph each correctly. The optimization can't be

    applied without regressing this test.

    Checked on x86_64, bug-strcoll2.c fails without this patch and passes

    after. This will also get a fix on 2.22 which has the same bug.

Index: glibc-2.22/locale/C-collate.c

===================================================================

--- glibc-2.22.orig/locale/C-collate.c

+++ glibc-2.22/locale/C-collate.c

@@ -144,8 +144,6 @@ const struct __locale_data _nl_C_LC_COLL

     /* _NL_COLLATE_COLLSEQWC */

     { .string = (const char *) collseqwc },

     /* _NL_COLLATE_CODESET */

-    { .string = _nl_C_codeset },

-    /* _NL_COLLATE_ENCODING_TYPE */

-    { .word = __cet_8bit }

+    { .string = _nl_C_codeset }

   }

 };

Index: glibc-2.22/locale/categories.def

===================================================================

--- glibc-2.22.orig/locale/categories.def

+++ glibc-2.22/locale/categories.def

@@ -58,7 +58,6 @@ DEFINE_CATEGORY

   DEFINE_ELEMENT (_NL_COLLATE_COLLSEQMB,        "collate-collseqmb",        std, wstring)

   DEFINE_ELEMENT (_NL_COLLATE_COLLSEQWC,        "collate-collseqwc",        std, wstring)

   DEFINE_ELEMENT (_NL_COLLATE_CODESET,		"collate-codeset",	    std, string)

-  DEFINE_ELEMENT (_NL_COLLATE_ENCODING_TYPE,   "collate-encoding-type",    std, word)

   ), NO_POSTLOAD)

Index: glibc-2.22/locale/langinfo.h

===================================================================

--- glibc-2.22.orig/locale/langinfo.h

+++ glibc-2.22/locale/langinfo.h

@@ -255,7 +255,6 @@ enum

   _NL_COLLATE_COLLSEQMB,

   _NL_COLLATE_COLLSEQWC,

   _NL_COLLATE_CODESET,

-  _NL_COLLATE_ENCODING_TYPE,

   _NL_NUM_LC_COLLATE,

   /* LC_CTYPE category: character classification.

Index: glibc-2.22/locale/localeinfo.h

===================================================================

--- glibc-2.22.orig/locale/localeinfo.h

+++ glibc-2.22/locale/localeinfo.h

@@ -110,14 +110,6 @@ enum coll_sort_rule

   sort_mask

 };

-/* Collation encoding type.  */

-enum collation_encoding_type

-{

-  __cet_other,

-  __cet_8bit,

-  __cet_utf8

-};

-

 /* We can map the types of the entries into a few categories.  */

 enum value_type

 {

Index: glibc-2.22/locale/programs/ld-collate.c

===================================================================

--- glibc-2.22.orig/locale/programs/ld-collate.c

+++ glibc-2.22/locale/programs/ld-collate.c

@@ -32,7 +32,6 @@

 #include "linereader.h"

 #include "locfile.h"

 #include "elem-hash.h"

-#include "../localeinfo.h"

 /* Uncomment the following line in the production version.  */

 /* #define NDEBUG 1 */

@@ -2131,8 +2130,6 @@ collate_output (struct localedef_t *loca

 	  /* The words have to be handled specially.  */

 	  if (idx == _NL_ITEM_INDEX (_NL_COLLATE_SYMB_HASH_SIZEMB))

 	    add_locale_uint32 (&file, 0);

-	  else if (idx == _NL_ITEM_INDEX (_NL_COLLATE_ENCODING_TYPE))

-	    add_locale_uint32 (&file, __cet_other);

 	  else

 	    add_locale_empty (&file;;

 	}

@@ -2496,12 +2493,6 @@ collate_output (struct localedef_t *loca

   add_locale_raw_data (&file, collate->mbseqorder, 256);

   add_locale_collseq_table (&file, &collate->wcseqorder);

   add_locale_string (&file, charmap->code_set_name);

-  if (strcmp (charmap->code_set_name, "UTF-8") == 0)

-    add_locale_uint32 (&file, __cet_utf8);

-  else if (charmap->mb_cur_max == 1)

-    add_locale_uint32 (&file, __cet_8bit);

-  else

-    add_locale_uint32 (&file, __cet_other);

   write_locale_data (output_path, LC_COLLATE, "LC_COLLATE", &file;;

   obstack_free (&weightpool, NULL);

Index: glibc-2.22/string/Makefile

===================================================================

--- glibc-2.22.orig/string/Makefile

+++ glibc-2.22/string/Makefile

@@ -54,7 +54,7 @@ tests		:= tester inl-tester noinl-tester

 		   tst-strtok tst-strxfrm bug-strcoll1 tst-strfry	\

 		   bug-strtok1 $(addprefix test-,$(strop-tests))	\

 		   bug-envz1 tst-strxfrm2 tst-endian tst-svc2		\

-		   tst-strtok_r

+		   tst-strtok_r bug-strcoll2

 xtests = tst-strcoll-overflow

@@ -75,4 +75,18 @@ ifeq ($(run-built-tests),yes)

 $(objpfx)tst-svc-cmp.out: tst-svc.expect $(objpfx)tst-svc.out

 	cmp $^ > $@; \

 	$(evaluate-test)

+

+LOCALES := de_DE.UTF-8 en_US.ISO-8859-1 en_US.UTF-8 \

+	   tr_TR.ISO-8859-9 tr_TR.UTF-8 cs_CZ.UTF-8 \

+	   da_DK.ISO-8859-1

+

+include ../gen-locales.mk

+

+$(objpfx)test-strcasecmp.out: $(gen-locales)

+$(objpfx)test-strncasecmp.out: $(gen-locales)

+$(objpfx)tst-strxfrm.out: $(gen-locales)

+$(objpfx)tst-strxfrm2.out: $(gen-locales)

+# bug-strcoll2 needs cs_CZ.UTF-8 and da_DK.ISO-8859-1.

+$(objpfx)bug-strcoll2.out: $(gen-locales)

+

 endif

Index: glibc-2.22/string/bug-strcoll2.c

===================================================================

--- /dev/null

+++ glibc-2.22/string/bug-strcoll2.c

@@ -0,0 +1,92 @@

+/* Bug 18589: sort-test.sh fails at random.

+   Copyright (C) 2015 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <stdio.h>

+#include <string.h>

+#include <locale.h>

+

+/* An incorrect strcoll optimization resulted in incorrect

+   results from strcoll for cs_CZ and da_DK.  */

+

+int

+test_cs_CZ (void)

+{

+  const char t1[] = "config";

+  const char t2[] = "choose";

+  if (setlocale (LC_ALL, "cs_CZ.UTF-8") == NULL)

+    {

+      perror ("setlocale");

+      return 1;

+    }

+  /* In Czech the digraph ch sorts after c, therefore we expect

+     config to sort before choose.  */

+  int a = strcoll (t1, t2);

+  int b = strcoll (t2, t1);

+  printf ("strcoll (\"%s\", \"%s\") = %d\n", t1, t2, a);

+  printf ("strcoll (\"%s\", \"%s\") = %d\n", t2, t1, b);

+  if (a < 0 && b > 0)

+    {

+      puts ("PASS: config < choose");

+      return 0;

+    }

+  else

+    {

+      puts ("FAIL: Wrong sorting in cs_CZ.UTF-8.");

+      return 1;

+    }

+}

+

+int

+test_da_DK (void)

+{

+  const char t1[] = "AS";

+  const char t2[] = "AA";

+  if (setlocale (LC_ALL, "da_DK.ISO-8859-1") == NULL)

+    {

+      perror ("setlocale");

+      return 1;

+    }

+  /* AA should be treated as the last letter of the Danish alphabet,

+     hence sorting after AS.  */

+  int a = strcoll (t1, t2);

+  int b = strcoll (t2, t1);

+  printf ("strcoll (\"%s\", \"%s\") = %d\n", t1, t2, a);

+  printf ("strcoll (\"%s\", \"%s\") = %d\n", t2, t1, b);

+  if (a < 0 && b > 0)

+    {

+      puts ("PASS: AS < AA");

+      return 0;

+    }

+  else

+    {

+      puts ("FAIL: Wrong sorting in da_DK.ISO-8859-1");

+      return 1;

+    }

+}

+

+static int

+do_test (void)

+{

+  int err = 0;

+  err |= test_cs_CZ ();

+  err |= test_da_DK ();

+  return err;

+}

+

+#define TEST_FUNCTION do_test ()

+#include "../test-skeleton.c"

Index: glibc-2.22/string/strcoll_l.c

===================================================================

--- glibc-2.22.orig/string/strcoll_l.c

+++ glibc-2.22/string/strcoll_l.c

@@ -29,7 +29,6 @@

 # define STRING_TYPE char

 # define USTRING_TYPE unsigned char

 # define STRCOLL __strcoll_l

-# define STRDIFF __strdiff

 # define STRCMP strcmp

 # define WEIGHT_H "../locale/weight.h"

 # define SUFFIX	MB

@@ -42,20 +41,6 @@

 #include "../locale/localeinfo.h"

 #include WEIGHT_H

-#define MASK_UTF8_7BIT  (1 << 7)

-#define MASK_UTF8_START (3 << 6)

-

-size_t

-STRDIFF (const STRING_TYPE *s, const STRING_TYPE *t)

-{

-  size_t n;

-

-  for (n = 0; *s != '\0' && *s++ == *t++; ++n)

-    continue;

-

-  return n;

-}

-

 /* Track status while looking for sequences in a string.  */

 typedef struct

 {

@@ -269,29 +254,9 @@ STRCOLL (const STRING_TYPE *s1, const ST

   const USTRING_TYPE *extra;

   const int32_t *indirect;

-  /* In case there is no locale specific sort order (C / POSIX).  */

   if (nrules == 0)

     return STRCMP (s1, s2);

-  /* Fast forward to the position of the first difference.  Needs to be

-     encoding aware as the byte-by-byte comparison can stop in the middle

-     of a char sequence for multibyte encodings like UTF-8.  */

-  uint_fast32_t encoding =

-    current->values[_NL_ITEM_INDEX (_NL_COLLATE_ENCODING_TYPE)].word;

-  if (encoding != __cet_other)

-    {

-      size_t diff = STRDIFF (s1, s2);

-      if (diff > 0)

-	{

-	  if (encoding == __cet_utf8 && (*(s1 + diff) & MASK_UTF8_7BIT) != 0)

-	    do

-	      diff--;

-	    while (diff > 0 && (*(s1 + diff) & MASK_UTF8_START) != MASK_UTF8_START);

-	  s1 += diff;

-	  s2 += diff;

-	}

-    }

-

   /* Catch empty strings.  */

   if (__glibc_unlikely (*s1 == '\0') || __glibc_unlikely (*s2 == '\0'))

     return (*s1 != '\0') - (*s2 != '\0');

@@ -358,8 +323,7 @@ STRCOLL (const STRING_TYPE *s1, const ST

 		     byte-level comparison to ensure that we don't waste time

 		     going through multiple passes for totally equal strings

 		     before proceeding to subsequent passes.  */

-		  if (pass == 0 && encoding == __cet_other &&

-		      STRCMP (s1, s2) == 0)

+		  if (pass == 0 && STRCMP (s1, s2) == 0)

 		    return result;

 		  else

 		    break;

Index: glibc-2.22/string/tst-strxfrm2.c

===================================================================

--- glibc-2.22.orig/string/tst-strxfrm2.c

+++ glibc-2.22/string/tst-strxfrm2.c

@@ -5,6 +5,8 @@

 static int

 do_test (void)

 {

+  static const char test_locale[] = "de_DE.UTF-8";

+

   int res = 0;

   char buf[20];

@@ -38,9 +40,9 @@ do_test (void)

       res = 1;

     }

-  if (setlocale (LC_ALL, "de_DE.UTF-8") == NULL)

+  if (setlocale (LC_ALL, test_locale) == NULL)

     {

-      puts ("setlocale failed");

+      printf ("cannot set locale \"%s\"\n", test_locale);

       res = 1;

     }

   else

Index: glibc-2.22/wcsmbs/wcscoll_l.c

===================================================================

--- glibc-2.22.orig/wcsmbs/wcscoll_l.c

+++ glibc-2.22/wcsmbs/wcscoll_l.c

@@ -23,7 +23,6 @@

 #define STRING_TYPE wchar_t

 #define USTRING_TYPE wint_t

 #define STRCOLL __wcscoll_l

-#define STRDIFF __wcsdiff

 #define STRCMP __wcscmp

 #define WEIGHT_H "../locale/weightwc.h"

 #define SUFFIX	WC