jacqueline
1 year ago
parent
18d90051c9
commit
b58c081508
|
@ -0,0 +1,11 @@ |
|||||||
|
# Copyright 2023 jacqueline <me@jacqueline.id.au> |
||||||
|
# |
||||||
|
# SPDX-License-Identifier: GPL-3.0-only |
||||||
|
|
||||||
|
idf_component_register( |
||||||
|
SRCS "collation.cpp" "strxfrm_l.c" |
||||||
|
INCLUDE_DIRS "include" |
||||||
|
PRIV_INCLUDE_DIRS "priv_include" |
||||||
|
REQUIRES "span" "esp_partition" "spi_flash") |
||||||
|
|
||||||
|
target_compile_options(${COMPONENT_LIB} PRIVATE ${EXTRA_WARNINGS}) |
||||||
@ -0,0 +1,92 @@ |
|||||||
|
/*
|
||||||
|
* Copyright 2023 jacqueline <me@jacqueline.id.au> |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: GPL-3.0-only |
||||||
|
*/ |
||||||
|
|
||||||
|
#include "collation.hpp" |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <memory> |
||||||
|
|
||||||
|
#include "esp_flash_spi_init.h" |
||||||
|
#include "esp_log.h" |
||||||
|
#include "esp_partition.h" |
||||||
|
#include "hal/spi_flash_types.h" |
||||||
|
#include "spi_flash_mmap.h" |
||||||
|
#include "strxfrm.h" |
||||||
|
|
||||||
|
namespace locale { |
||||||
|
|
||||||
|
static constexpr char kTag[] = "collate"; |
||||||
|
|
||||||
|
static constexpr esp_partition_type_t kLocalePartitionType = |
||||||
|
static_cast<esp_partition_type_t>(0x40); |
||||||
|
static constexpr esp_partition_subtype_t kLcCollateSubtype = |
||||||
|
static_cast<esp_partition_subtype_t>(0x0); |
||||||
|
|
||||||
|
auto CreateCollator() -> std::unique_ptr<ICollator> { |
||||||
|
std::unique_ptr<ICollator> ret{GLibCollator::create()}; |
||||||
|
if (!ret) { |
||||||
|
ret.reset(new NoopCollator()); |
||||||
|
} |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
auto GLibCollator::create() -> GLibCollator* { |
||||||
|
uint32_t data_pages = spi_flash_mmap_get_free_pages(SPI_FLASH_MMAP_DATA); |
||||||
|
ESP_LOGI(kTag, "free data pages: %lu (%lu KiB)", data_pages, data_pages * 64); |
||||||
|
|
||||||
|
const esp_partition_t* partition = |
||||||
|
esp_partition_find_first(kLocalePartitionType, kLcCollateSubtype, NULL); |
||||||
|
if (partition == NULL) { |
||||||
|
ESP_LOGW(kTag, "no LC_COLLATE partition found"); |
||||||
|
} |
||||||
|
|
||||||
|
ESP_LOGI(kTag, "found LC_COLLATE partition of size %lu", partition->size); |
||||||
|
if (partition->size > data_pages * 64 * 1024) { |
||||||
|
ESP_LOGE(kTag, "not enough free pages to map LC_COLLATE partition!"); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
const void* region; |
||||||
|
esp_partition_mmap_handle_t handle; |
||||||
|
esp_err_t err = esp_partition_mmap(partition, 0, partition->size, |
||||||
|
ESP_PARTITION_MMAP_DATA, ®ion, &handle); |
||||||
|
if (err != ESP_OK) { |
||||||
|
ESP_LOGE(kTag, "LC_COLLATE mmap failed"); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
auto data = std::make_unique<locale_data_t>(); |
||||||
|
if (!parse_locale_data(region, partition->size, data.get())) { |
||||||
|
ESP_LOGE(kTag, "parsing locale data failed"); |
||||||
|
esp_partition_munmap(handle); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
return new GLibCollator(handle, std::move(data)); |
||||||
|
} |
||||||
|
|
||||||
|
GLibCollator::GLibCollator(const esp_partition_mmap_handle_t handle, |
||||||
|
std::unique_ptr<locale_data_t> locale) |
||||||
|
: handle_(handle), locale_data_(std::move(locale)) {} |
||||||
|
|
||||||
|
GLibCollator::~GLibCollator() { |
||||||
|
esp_partition_munmap(handle_); |
||||||
|
} |
||||||
|
|
||||||
|
auto GLibCollator::Transform(const std::string& in) -> std::string { |
||||||
|
char dest[256]; |
||||||
|
size_t size = glib_strxfrm(dest, in.c_str(), 256, locale_data_.get()); |
||||||
|
if (size >= 256) { |
||||||
|
char* larger_dest = new char[size + 1]{0}; |
||||||
|
glib_strxfrm(larger_dest, in.c_str(), size, locale_data_.get()); |
||||||
|
std::string out{larger_dest, size}; |
||||||
|
delete[] larger_dest; |
||||||
|
return out; |
||||||
|
} |
||||||
|
return {dest, size}; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace locale
|
||||||
@ -0,0 +1,49 @@ |
|||||||
|
/*
|
||||||
|
* Copyright 2023 jacqueline <me@jacqueline.id.au> |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: GPL-3.0-only |
||||||
|
*/ |
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <cstddef> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "esp_partition.h" |
||||||
|
#include "span.hpp" |
||||||
|
|
||||||
|
#include "strxfrm.h" |
||||||
|
|
||||||
|
namespace locale { |
||||||
|
|
||||||
|
class ICollator { |
||||||
|
public: |
||||||
|
virtual ~ICollator() {} |
||||||
|
|
||||||
|
virtual auto Transform(const std::string&) -> std::string = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
class NoopCollator : public ICollator { |
||||||
|
public: |
||||||
|
auto Transform(const std::string& in) -> std::string override { return in; } |
||||||
|
}; |
||||||
|
|
||||||
|
auto CreateCollator() -> std::unique_ptr<ICollator>; |
||||||
|
|
||||||
|
class GLibCollator : public ICollator { |
||||||
|
public: |
||||||
|
static auto create() -> GLibCollator*; |
||||||
|
~GLibCollator(); |
||||||
|
|
||||||
|
auto Transform(const std::string& in) -> std::string override; |
||||||
|
|
||||||
|
private: |
||||||
|
GLibCollator(const esp_partition_mmap_handle_t, |
||||||
|
std::unique_ptr<locale_data_t>); |
||||||
|
|
||||||
|
const esp_partition_mmap_handle_t handle_; |
||||||
|
std::unique_ptr<locale_data_t> locale_data_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace locale
|
||||||
@ -0,0 +1,35 @@ |
|||||||
|
/*
|
||||||
|
* Copyright 2023 jacqueline <me@jacqueline.id.au> |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: GPL-3.0-only |
||||||
|
*/ |
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <stdbool.h> |
||||||
|
|
||||||
|
#ifdef __cplusplus |
||||||
|
extern "C" { |
||||||
|
#endif |
||||||
|
|
||||||
|
typedef struct { |
||||||
|
uint_fast32_t nrules; |
||||||
|
unsigned char* rulesets; |
||||||
|
unsigned char* weights; |
||||||
|
int32_t* table; |
||||||
|
unsigned char* extra; |
||||||
|
int32_t* indirect; |
||||||
|
} locale_data_t; |
||||||
|
|
||||||
|
bool parse_locale_data(const void* raw_data, size_t size, locale_data_t* out); |
||||||
|
|
||||||
|
size_t glib_strxfrm(char* dest, |
||||||
|
const char* src, |
||||||
|
size_t n, |
||||||
|
locale_data_t* locale); |
||||||
|
|
||||||
|
#ifdef __cplusplus |
||||||
|
} |
||||||
|
#endif |
||||||
@ -0,0 +1,160 @@ |
|||||||
|
/* Copyright (C) 1996-2018 Free Software Foundation, Inc.
|
||||||
|
This file is part of the GNU C Library. |
||||||
|
Written by Ulrich Drepper, <drepper@cygnus.com>. |
||||||
|
|
||||||
|
The GNU C Library is free software; you can redistribute it and/or |
||||||
|
modify it under the terms of the GNU Lesser General Public |
||||||
|
License as published by the Free Software Foundation; either |
||||||
|
version 2.1 of the License, or (at your option) any later version. |
||||||
|
|
||||||
|
The GNU C Library is distributed in the hope that it will be useful, |
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
||||||
|
Lesser General Public License for more details. |
||||||
|
|
||||||
|
You should have received a copy of the GNU Lesser General Public |
||||||
|
License along with the GNU C Library; if not, see |
||||||
|
<http://www.gnu.org/licenses/>. */
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <stddef.h> |
||||||
|
|
||||||
|
/* This alignment is used for 32-bit integers in locale files, both
|
||||||
|
those that are explicitly int32_t or uint32_t and those that are |
||||||
|
wchar_t, regardless of the (possibly smaller) alignment required |
||||||
|
for such integers on a particular host. */ |
||||||
|
#define LOCFILE_ALIGN sizeof (int32_t) |
||||||
|
#define LOCFILE_ALIGN_MASK (LOCFILE_ALIGN - 1) |
||||||
|
#define LOCFILE_ALIGN_UP(x) (((x) + LOCFILE_ALIGN - 1) \ |
||||||
|
& ~LOCFILE_ALIGN_MASK) |
||||||
|
#define LOCFILE_ALIGNED_P(x) (((x) & LOCFILE_ALIGN_MASK) == 0) |
||||||
|
|
||||||
|
/* Find index of weight. */ |
||||||
|
static inline int32_t __attribute__ ((always_inline)) |
||||||
|
findidx (const int32_t *table, |
||||||
|
const int32_t *indirect, |
||||||
|
const unsigned char *extra, |
||||||
|
const unsigned char **cpp, size_t len) |
||||||
|
{ |
||||||
|
int_fast32_t i = table[*(*cpp)++]; |
||||||
|
const unsigned char *cp; |
||||||
|
const unsigned char *usrc; |
||||||
|
|
||||||
|
if (i >= 0) |
||||||
|
/* This is an index into the weight table. Cool. */ |
||||||
|
return i; |
||||||
|
|
||||||
|
/* Oh well, more than one sequence starting with this byte.
|
||||||
|
Search for the correct one. */ |
||||||
|
cp = &extra[-i]; |
||||||
|
usrc = *cpp; |
||||||
|
--len; |
||||||
|
while (1) |
||||||
|
{ |
||||||
|
size_t nhere; |
||||||
|
|
||||||
|
/* The first thing is the index. */ |
||||||
|
i = *((const int32_t *) cp); |
||||||
|
cp += sizeof (int32_t); |
||||||
|
|
||||||
|
/* Next is the length of the byte sequence. These are always
|
||||||
|
short byte sequences so there is no reason to call any |
||||||
|
function (even if they are inlined). */ |
||||||
|
nhere = *cp++; |
||||||
|
|
||||||
|
if (i >= 0) |
||||||
|
{ |
||||||
|
/* It is a single character. If it matches we found our
|
||||||
|
index. Note that at the end of each list there is an |
||||||
|
entry of length zero which represents the single byte |
||||||
|
sequence. The first (and here only) byte was tested |
||||||
|
already. */ |
||||||
|
size_t cnt; |
||||||
|
|
||||||
|
/* With GCC 5.3 when compiling with -Os the compiler warns
|
||||||
|
that seq2.back_us, which becomes usrc, might be used |
||||||
|
uninitialized. This can't be true because we pass a length |
||||||
|
of -1 for len at the same time which means that this loop |
||||||
|
never executes. */ |
||||||
|
for (cnt = 0; cnt < nhere && cnt < len; ++cnt) |
||||||
|
if (cp[cnt] != usrc[cnt]) |
||||||
|
break; |
||||||
|
|
||||||
|
if (cnt == nhere) |
||||||
|
{ |
||||||
|
/* Found it. */ |
||||||
|
*cpp += nhere; |
||||||
|
return i; |
||||||
|
} |
||||||
|
|
||||||
|
/* Up to the next entry. */ |
||||||
|
cp += nhere; |
||||||
|
if (!LOCFILE_ALIGNED_P (1 + nhere)) |
||||||
|
cp += LOCFILE_ALIGN - (1 + nhere) % LOCFILE_ALIGN; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
/* This is a range of characters. First decide whether the
|
||||||
|
current byte sequence lies in the range. */ |
||||||
|
size_t cnt; |
||||||
|
size_t offset = 0; |
||||||
|
|
||||||
|
for (cnt = 0; cnt < nhere && cnt < len; ++cnt) |
||||||
|
if (cp[cnt] != usrc[cnt]) |
||||||
|
break; |
||||||
|
|
||||||
|
if (cnt != nhere) |
||||||
|
{ |
||||||
|
if (cnt == len || cp[cnt] > usrc[cnt]) |
||||||
|
{ |
||||||
|
/* Cannot be in this range. */ |
||||||
|
cp += 2 * nhere; |
||||||
|
if (!LOCFILE_ALIGNED_P (1 + 2 * nhere)) |
||||||
|
cp += (LOCFILE_ALIGN |
||||||
|
- (1 + 2 * nhere) % LOCFILE_ALIGN); |
||||||
|
continue; |
||||||
|
} |
||||||
|
|
||||||
|
/* Test against the end of the range. */ |
||||||
|
for (cnt = 0; cnt < nhere; ++cnt) |
||||||
|
if (cp[nhere + cnt] != usrc[cnt]) |
||||||
|
break; |
||||||
|
|
||||||
|
if (cnt != nhere && cp[nhere + cnt] < usrc[cnt]) |
||||||
|
{ |
||||||
|
/* Cannot be in this range. */ |
||||||
|
cp += 2 * nhere; |
||||||
|
if (!LOCFILE_ALIGNED_P (1 + 2 * nhere)) |
||||||
|
cp += (LOCFILE_ALIGN |
||||||
|
- (1 + 2 * nhere) % LOCFILE_ALIGN); |
||||||
|
continue; |
||||||
|
} |
||||||
|
|
||||||
|
/* This range matches the next characters. Now find
|
||||||
|
the offset in the indirect table. */ |
||||||
|
for (cnt = 0; cp[cnt] == usrc[cnt]; ++cnt); |
||||||
|
|
||||||
|
do |
||||||
|
{ |
||||||
|
offset <<= 8; |
||||||
|
/* With GCC 7 when compiling with -Os the compiler
|
||||||
|
warns that seq1.back_us and seq2.back_us, which |
||||||
|
become usrc, might be used uninitialized. This |
||||||
|
is impossible for the same reason as described |
||||||
|
above. */ |
||||||
|
offset += usrc[cnt] - cp[cnt]; |
||||||
|
} |
||||||
|
while (++cnt < nhere); |
||||||
|
} |
||||||
|
|
||||||
|
*cpp += nhere; |
||||||
|
return indirect[-i + offset]; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* NOTREACHED */ |
||||||
|
return 0x43219876; |
||||||
|
} |
||||||
|
|
||||||
@ -0,0 +1,641 @@ |
|||||||
|
/* Copyright (C) 1995-2018 Free Software Foundation, Inc.
|
||||||
|
This file is part of the GNU C Library. |
||||||
|
Written by Ulrich Drepper <drepper@gnu.org>, 1995. |
||||||
|
|
||||||
|
The GNU C Library is free software; you can redistribute it and/or |
||||||
|
modify it under the terms of the GNU Lesser General Public |
||||||
|
License as published by the Free Software Foundation; either |
||||||
|
version 2.1 of the License, or (at your option) any later version. |
||||||
|
|
||||||
|
The GNU C Library is distributed in the hope that it will be useful, |
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
||||||
|
Lesser General Public License for more details. |
||||||
|
|
||||||
|
You should have received a copy of the GNU Lesser General Public |
||||||
|
License along with the GNU C Library; if not, see |
||||||
|
<http://www.gnu.org/licenses/>. */
|
||||||
|
|
||||||
|
#include "strxfrm.h" |
||||||
|
|
||||||
|
#include <assert.h> |
||||||
|
#include <stdbool.h> |
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <stdlib.h> |
||||||
|
#include <string.h> |
||||||
|
#include <sys/param.h> |
||||||
|
|
||||||
|
#include "esp_log.h" |
||||||
|
|
||||||
|
#include "weight.h" |
||||||
|
|
||||||
|
static const char kTag[] = "strxfrm"; |
||||||
|
|
||||||
|
#ifndef STRING_TYPE |
||||||
|
#define STRING_TYPE char |
||||||
|
#define USTRING_TYPE unsigned char |
||||||
|
#define STRLEN strlen |
||||||
|
#define STPNCPY stpncpy |
||||||
|
#define L(arg) arg |
||||||
|
#endif |
||||||
|
|
||||||
|
#define CONCAT(a, b) CONCAT1(a, b) |
||||||
|
#define CONCAT1(a, b) a##b |
||||||
|
|
||||||
|
/* Maximum string size that is calculated with cached indices. Right now this
|
||||||
|
is an arbitrary value open to optimizations. SMALL_STR_SIZE * 4 has to be |
||||||
|
lower than __MAX_ALLOCA_CUTOFF. Keep localedata/xfrm-test.c in sync. */ |
||||||
|
#define SMALL_STR_SIZE 4095 |
||||||
|
|
||||||
|
/* We know three kinds of collation sorting rules. */ |
||||||
|
enum coll_sort_rule { |
||||||
|
illegal_0__, |
||||||
|
sort_forward, |
||||||
|
sort_backward, |
||||||
|
illegal_3__, |
||||||
|
sort_position, |
||||||
|
sort_forward_position, |
||||||
|
sort_backward_position, |
||||||
|
sort_mask |
||||||
|
}; |
||||||
|
|
||||||
|
enum collate_element { |
||||||
|
COLLATE_NRULES = 0, |
||||||
|
COLLATE_RULESETS, |
||||||
|
COLLATE_TABLEMB, |
||||||
|
COLLATE_WEIGHTMB, |
||||||
|
COLLATE_EXTRAMB, |
||||||
|
COLLATE_INDIRECTMB, |
||||||
|
COLLATE_GAP1, |
||||||
|
COLLATE_GAP2, |
||||||
|
COLLATE_GAP3, |
||||||
|
COLLATE_TABLEWC, |
||||||
|
COLLATE_WEIGHTWC, |
||||||
|
COLLATE_EXTRAWC, |
||||||
|
COLLATE_INDIRECTWC, |
||||||
|
COLLATE_SYMB_HASH_SIZEMB, |
||||||
|
COLLATE_SYMB_TABLEMB, |
||||||
|
COLLATE_SYMB_EXTRAMB, |
||||||
|
COLLATE_COLLSEQMB, |
||||||
|
COLLATE_COLLSEQWC, |
||||||
|
COLLATE_CODESET, |
||||||
|
|
||||||
|
// Not a real element; used to know how many elements there are.
|
||||||
|
COLLATE_LAST |
||||||
|
}; |
||||||
|
typedef enum collate_element collate_element_t; |
||||||
|
|
||||||
|
bool parse_locale_data(const void* raw_data, size_t size, locale_data_t* out) { |
||||||
|
const struct { |
||||||
|
unsigned int magic; |
||||||
|
unsigned int nstrings; |
||||||
|
unsigned int strindex[0]; |
||||||
|
}* const header = raw_data; |
||||||
|
|
||||||
|
if (header->magic != 0x20051017) { |
||||||
|
ESP_LOGE(kTag, "file magic incorrect (was %x)", header->magic); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (sizeof(*header) + header->nstrings * sizeof(unsigned int) >= size) { |
||||||
|
ESP_LOGE(kTag, "file was too small to contain header"); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (header->nstrings != COLLATE_LAST) { |
||||||
|
ESP_LOGE(kTag, "file has incorrect number of elements (was %u, wanted %u)", |
||||||
|
header->nstrings, COLLATE_LAST); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
// The LC_COLLATE partition appears to contain data in the correct shape.
|
||||||
|
// Pull out pointers to the various attributes it contains.
|
||||||
|
const void* offsets[COLLATE_LAST]; |
||||||
|
for (size_t i = 0; i < header->nstrings; i++) { |
||||||
|
size_t offset = header->strindex[i]; |
||||||
|
if (offset > size) { |
||||||
|
ESP_LOGE(kTag, "element offset (%u) exceeds file size", offset); |
||||||
|
return false; |
||||||
|
} |
||||||
|
offsets[i] = (raw_data + offset); |
||||||
|
} |
||||||
|
|
||||||
|
// Now parse those pointers into the output struct.
|
||||||
|
out->nrules = *(const unsigned int*)offsets[COLLATE_NRULES]; |
||||||
|
|
||||||
|
out->rulesets = (unsigned char*)offsets[COLLATE_RULESETS]; |
||||||
|
out->table = (int32_t*)offsets[COLLATE_TABLEMB]; |
||||||
|
out->weights = (unsigned char*)offsets[COLLATE_WEIGHTMB]; |
||||||
|
out->extra = (unsigned char*)offsets[COLLATE_EXTRAMB]; |
||||||
|
out->indirect = (int32_t*)offsets[COLLATE_INDIRECTMB]; |
||||||
|
|
||||||
|
assert(((uintptr_t)out->table) % __alignof__(out->table[0]) == 0); |
||||||
|
assert(((uintptr_t)out->weights) % __alignof__(out->weights[0]) == 0); |
||||||
|
assert(((uintptr_t)out->extra) % __alignof__(out->extra[0]) == 0); |
||||||
|
assert(((uintptr_t)out->indirect) % __alignof__(out->indirect[0]) == 0); |
||||||
|
|
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
/* We need UTF-8 encoding of numbers. */ |
||||||
|
static int utf8_encode(char* buf, int val) { |
||||||
|
int retval; |
||||||
|
|
||||||
|
if (val < 0x80) { |
||||||
|
*buf++ = (char)val; |
||||||
|
retval = 1; |
||||||
|
} else { |
||||||
|
int step; |
||||||
|
|
||||||
|
for (step = 2; step < 6; ++step) |
||||||
|
if ((val & (~(uint32_t)0 << (5 * step + 1))) == 0) |
||||||
|
break; |
||||||
|
retval = step; |
||||||
|
|
||||||
|
*buf = (unsigned char)(~0xff >> step); |
||||||
|
--step; |
||||||
|
do { |
||||||
|
buf[step] = 0x80 | (val & 0x3f); |
||||||
|
val >>= 6; |
||||||
|
} while (--step > 0); |
||||||
|
*buf |= val; |
||||||
|
} |
||||||
|
|
||||||
|
return retval; |
||||||
|
} |
||||||
|
|
||||||
|
/* Find next weight and rule index. Inlined since called for every char. */ |
||||||
|
static __always_inline size_t find_idx(const USTRING_TYPE** us, |
||||||
|
int32_t* weight_idx, |
||||||
|
unsigned char* rule_idx, |
||||||
|
const locale_data_t* l_data, |
||||||
|
const int pass) { |
||||||
|
int32_t tmp = findidx(l_data->table, l_data->indirect, l_data->extra, us, -1); |
||||||
|
*rule_idx = tmp >> 24; |
||||||
|
int32_t idx = tmp & 0xffffff; |
||||||
|
size_t len = l_data->weights[idx++]; |
||||||
|
|
||||||
|
/* Skip over indices of previous levels. */ |
||||||
|
for (int i = 0; i < pass; i++) { |
||||||
|
idx += len; |
||||||
|
len = l_data->weights[idx++]; |
||||||
|
} |
||||||
|
|
||||||
|
*weight_idx = idx; |
||||||
|
return len; |
||||||
|
} |
||||||
|
|
||||||
|
static int find_position(const USTRING_TYPE* us, |
||||||
|
const locale_data_t* l_data, |
||||||
|
const int pass) { |
||||||
|
int32_t weight_idx; |
||||||
|
unsigned char rule_idx; |
||||||
|
const USTRING_TYPE* usrc = us; |
||||||
|
|
||||||
|
find_idx(&usrc, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
return l_data->rulesets[rule_idx * l_data->nrules + pass] & sort_position; |
||||||
|
} |
||||||
|
|
||||||
|
/* Do the transformation. */ |
||||||
|
static size_t do_xfrm(const USTRING_TYPE* usrc, |
||||||
|
STRING_TYPE* dest, |
||||||
|
size_t n, |
||||||
|
const locale_data_t* l_data) { |
||||||
|
int32_t weight_idx; |
||||||
|
unsigned char rule_idx; |
||||||
|
uint_fast32_t pass; |
||||||
|
size_t needed = 0; |
||||||
|
size_t last_needed; |
||||||
|
|
||||||
|
/* Now the passes over the weights. */ |
||||||
|
for (pass = 0; pass < l_data->nrules; ++pass) { |
||||||
|
size_t backw_len = 0; |
||||||
|
last_needed = needed; |
||||||
|
const USTRING_TYPE* cur = usrc; |
||||||
|
const USTRING_TYPE* backw_start = NULL; |
||||||
|
|
||||||
|
/* We assume that if a rule has defined `position' in one section
|
||||||
|
this is true for all of them. */ |
||||||
|
int position = find_position(cur, l_data, pass); |
||||||
|
|
||||||
|
if (position == 0) { |
||||||
|
while (*cur != L('\0')) { |
||||||
|
const USTRING_TYPE* pos = cur; |
||||||
|
size_t len = find_idx(&cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
int rule = l_data->rulesets[rule_idx * l_data->nrules + pass]; |
||||||
|
|
||||||
|
if ((rule & sort_forward) != 0) { |
||||||
|
/* Handle the pushed backward sequence. */ |
||||||
|
if (backw_start != NULL) { |
||||||
|
for (size_t i = backw_len; i > 0;) { |
||||||
|
int32_t weight_idx; |
||||||
|
unsigned char rule_idx; |
||||||
|
size_t len = |
||||||
|
find_idx(&backw_start, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
if (needed + i < n) |
||||||
|
for (size_t j = len; j > 0; j--) |
||||||
|
dest[needed + i - j] = l_data->weights[weight_idx++]; |
||||||
|
|
||||||
|
i -= len; |
||||||
|
} |
||||||
|
|
||||||
|
needed += backw_len; |
||||||
|
backw_start = NULL; |
||||||
|
backw_len = 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* Now handle the forward element. */ |
||||||
|
if (needed + len < n) |
||||||
|
while (len-- > 0) |
||||||
|
dest[needed++] = l_data->weights[weight_idx++]; |
||||||
|
else |
||||||
|
/* No more characters fit into the buffer. */ |
||||||
|
needed += len; |
||||||
|
} else { |
||||||
|
/* Remember start of the backward sequence & track length. */ |
||||||
|
if (backw_start == NULL) |
||||||
|
backw_start = pos; |
||||||
|
backw_len += len; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* Handle the pushed backward sequence. */ |
||||||
|
if (backw_start != NULL) { |
||||||
|
for (size_t i = backw_len; i > 0;) { |
||||||
|
size_t len = |
||||||
|
find_idx(&backw_start, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
if (needed + i < n) |
||||||
|
for (size_t j = len; j > 0; j--) |
||||||
|
dest[needed + i - j] = l_data->weights[weight_idx++]; |
||||||
|
|
||||||
|
i -= len; |
||||||
|
} |
||||||
|
|
||||||
|
needed += backw_len; |
||||||
|
} |
||||||
|
} else { |
||||||
|
int val = 1; |
||||||
|
char buf[7]; |
||||||
|
size_t buflen; |
||||||
|
size_t i; |
||||||
|
|
||||||
|
while (*cur != L('\0')) { |
||||||
|
const USTRING_TYPE* pos = cur; |
||||||
|
size_t len = find_idx(&cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
int rule = l_data->rulesets[rule_idx * l_data->nrules + pass]; |
||||||
|
|
||||||
|
if ((rule & sort_forward) != 0) { |
||||||
|
/* Handle the pushed backward sequence. */ |
||||||
|
if (backw_start != NULL) { |
||||||
|
for (size_t p = backw_len; p > 0; p--) { |
||||||
|
size_t len; |
||||||
|
int32_t weight_idx; |
||||||
|
unsigned char rule_idx; |
||||||
|
const USTRING_TYPE* backw_cur = backw_start; |
||||||
|
|
||||||
|
/* To prevent a warning init the used vars. */ |
||||||
|
len = find_idx(&backw_cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
|
||||||
|
for (i = 1; i < p; i++) |
||||||
|
len = |
||||||
|
find_idx(&backw_cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
|
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = l_data->weights[weight_idx + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
++val; |
||||||
|
} |
||||||
|
|
||||||
|
backw_start = NULL; |
||||||
|
backw_len = 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* Now handle the forward element. */ |
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = l_data->weights[weight_idx + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
++val; |
||||||
|
} else { |
||||||
|
/* Remember start of the backward sequence & track length. */ |
||||||
|
if (backw_start == NULL) |
||||||
|
backw_start = pos; |
||||||
|
backw_len++; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* Handle the pushed backward sequence. */ |
||||||
|
if (backw_start != NULL) { |
||||||
|
for (size_t p = backw_len; p > 0; p--) { |
||||||
|
size_t len; |
||||||
|
int32_t weight_idx; |
||||||
|
unsigned char rule_idx; |
||||||
|
const USTRING_TYPE* backw_cur = backw_start; |
||||||
|
|
||||||
|
/* To prevent a warning init the used vars. */ |
||||||
|
len = find_idx(&backw_cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
|
||||||
|
for (i = 1; i < p; i++) |
||||||
|
len = find_idx(&backw_cur, &weight_idx, &rule_idx, l_data, pass); |
||||||
|
|
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = l_data->weights[weight_idx + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
++val; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* Finally store the byte to separate the passes or terminate
|
||||||
|
the string. */ |
||||||
|
if (needed < n) |
||||||
|
dest[needed] = pass + 1 < l_data->nrules ? L('\1') : L('\0'); |
||||||
|
++needed; |
||||||
|
} |
||||||
|
|
||||||
|
/* This is a little optimization: many collation specifications have
|
||||||
|
a `position' rule at the end and if no non-ignored character |
||||||
|
is found the last \1 byte is immediately followed by a \0 byte |
||||||
|
signalling this. We can avoid the \1 byte(s). */ |
||||||
|
if (needed > 2 && needed == last_needed + 1) { |
||||||
|
/* Remove the \1 byte. */ |
||||||
|
if (--needed <= n) |
||||||
|
dest[needed - 1] = L('\0'); |
||||||
|
} |
||||||
|
|
||||||
|
/* Return the number of bytes/words we need, but don't count the NUL
|
||||||
|
byte/word at the end. */ |
||||||
|
return needed - 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* Do the transformation using weight-index and rule cache. */ |
||||||
|
static size_t do_xfrm_cached(STRING_TYPE* dest, |
||||||
|
size_t n, |
||||||
|
const locale_data_t* l_data, |
||||||
|
size_t idxmax, |
||||||
|
int32_t* idxarr, |
||||||
|
const unsigned char* rulearr) { |
||||||
|
uint_fast32_t nrules = l_data->nrules; |
||||||
|
unsigned char* rulesets = l_data->rulesets; |
||||||
|
USTRING_TYPE* weights = l_data->weights; |
||||||
|
uint_fast32_t pass; |
||||||
|
size_t needed = 0; |
||||||
|
size_t last_needed; |
||||||
|
size_t idxcnt; |
||||||
|
|
||||||
|
/* Now the passes over the weights. */ |
||||||
|
for (pass = 0; pass < nrules; ++pass) { |
||||||
|
size_t backw_stop = ~0ul; |
||||||
|
int rule = rulesets[rulearr[0] * nrules + pass]; |
||||||
|
/* We assume that if a rule has defined `position' in one section
|
||||||
|
this is true for all of them. */ |
||||||
|
int position = rule & sort_position; |
||||||
|
|
||||||
|
last_needed = needed; |
||||||
|
if (position == 0) { |
||||||
|
for (idxcnt = 0; idxcnt < idxmax; ++idxcnt) { |
||||||
|
if ((rule & sort_forward) != 0) { |
||||||
|
size_t len; |
||||||
|
|
||||||
|
if (backw_stop != ~0ul) { |
||||||
|
/* Handle the pushed elements now. */ |
||||||
|
size_t backw; |
||||||
|
|
||||||
|
for (backw = idxcnt; backw > backw_stop;) { |
||||||
|
--backw; |
||||||
|
len = weights[idxarr[backw]++]; |
||||||
|
|
||||||
|
if (needed + len < n) |
||||||
|
while (len-- > 0) |
||||||
|
dest[needed++] = weights[idxarr[backw]++]; |
||||||
|
else { |
||||||
|
/* No more characters fit into the buffer. */ |
||||||
|
needed += len; |
||||||
|
idxarr[backw] += len; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
backw_stop = ~0ul; |
||||||
|
} |
||||||
|
|
||||||
|
/* Now handle the forward element. */ |
||||||
|
len = weights[idxarr[idxcnt]++]; |
||||||
|
if (needed + len < n) |
||||||
|
while (len-- > 0) |
||||||
|
dest[needed++] = weights[idxarr[idxcnt]++]; |
||||||
|
else { |
||||||
|
/* No more characters fit into the buffer. */ |
||||||
|
needed += len; |
||||||
|
idxarr[idxcnt] += len; |
||||||
|
} |
||||||
|
} else { |
||||||
|
/* Remember where the backwards series started. */ |
||||||
|
if (backw_stop == ~0ul) |
||||||
|
backw_stop = idxcnt; |
||||||
|
} |
||||||
|
|
||||||
|
rule = rulesets[rulearr[idxcnt + 1] * nrules + pass]; |
||||||
|
} |
||||||
|
|
||||||
|
if (backw_stop != ~0ul) { |
||||||
|
/* Handle the pushed elements now. */ |
||||||
|
size_t backw; |
||||||
|
|
||||||
|
backw = idxcnt; |
||||||
|
while (backw > backw_stop) { |
||||||
|
size_t len = weights[idxarr[--backw]++]; |
||||||
|
|
||||||
|
if (needed + len < n) |
||||||
|
while (len-- > 0) |
||||||
|
dest[needed++] = weights[idxarr[backw]++]; |
||||||
|
else { |
||||||
|
/* No more characters fit into the buffer. */ |
||||||
|
needed += len; |
||||||
|
idxarr[backw] += len; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} else { |
||||||
|
int val = 1; |
||||||
|
char buf[7]; |
||||||
|
size_t buflen; |
||||||
|
size_t i; |
||||||
|
|
||||||
|
for (idxcnt = 0; idxcnt < idxmax; ++idxcnt) { |
||||||
|
if ((rule & sort_forward) != 0) { |
||||||
|
size_t len; |
||||||
|
|
||||||
|
if (backw_stop != ~0ul) { |
||||||
|
/* Handle the pushed elements now. */ |
||||||
|
size_t backw; |
||||||
|
|
||||||
|
for (backw = idxcnt; backw > backw_stop;) { |
||||||
|
--backw; |
||||||
|
len = weights[idxarr[backw]++]; |
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = weights[idxarr[backw] + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
idxarr[backw] += len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
++val; |
||||||
|
} |
||||||
|
|
||||||
|
backw_stop = ~0ul; |
||||||
|
} |
||||||
|
|
||||||
|
/* Now handle the forward element. */ |
||||||
|
len = weights[idxarr[idxcnt]++]; |
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = weights[idxarr[idxcnt] + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
idxarr[idxcnt] += len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
/* Note that we don't have to increment `idxarr[idxcnt]'
|
||||||
|
since the length is zero. */ |
||||||
|
++val; |
||||||
|
} else { |
||||||
|
/* Remember where the backwards series started. */ |
||||||
|
if (backw_stop == ~0ul) |
||||||
|
backw_stop = idxcnt; |
||||||
|
} |
||||||
|
|
||||||
|
rule = rulesets[rulearr[idxcnt + 1] * nrules + pass]; |
||||||
|
} |
||||||
|
|
||||||
|
if (backw_stop != ~0ul) { |
||||||
|
/* Handle the pushed elements now. */ |
||||||
|
size_t backw; |
||||||
|
|
||||||
|
backw = idxmax - 1; |
||||||
|
while (backw > backw_stop) { |
||||||
|
size_t len = weights[idxarr[--backw]++]; |
||||||
|
if (len != 0) { |
||||||
|
buflen = utf8_encode(buf, val); |
||||||
|
if (needed + buflen + len < n) { |
||||||
|
for (i = 0; i < buflen; ++i) |
||||||
|
dest[needed + i] = buf[i]; |
||||||
|
for (i = 0; i < len; ++i) |
||||||
|
dest[needed + buflen + i] = weights[idxarr[backw] + i]; |
||||||
|
} |
||||||
|
needed += buflen + len; |
||||||
|
idxarr[backw] += len; |
||||||
|
val = 1; |
||||||
|
} else |
||||||
|
++val; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* Finally store the byte to separate the passes or terminate
|
||||||
|
the string. */ |
||||||
|
if (needed < n) |
||||||
|
dest[needed] = pass + 1 < nrules ? L('\1') : L('\0'); |
||||||
|
++needed; |
||||||
|
} |
||||||
|
|
||||||
|
/* This is a little optimization: many collation specifications have
|
||||||
|
a `position' rule at the end and if no non-ignored character |
||||||
|
is found the last \1 byte is immediately followed by a \0 byte |
||||||
|
signalling this. We can avoid the \1 byte(s). */ |
||||||
|
if (needed > 2 && needed == last_needed + 1) { |
||||||
|
/* Remove the \1 byte. */ |
||||||
|
if (--needed <= n) |
||||||
|
dest[needed - 1] = L('\0'); |
||||||
|
} |
||||||
|
|
||||||
|
/* Return the number of bytes/words we need, but don't count the NUL
|
||||||
|
byte/word at the end. */ |
||||||
|
return needed - 1; |
||||||
|
} |
||||||
|
|
||||||
|
size_t glib_strxfrm(char* dest, |
||||||
|
const char* src, |
||||||
|
size_t n, |
||||||
|
locale_data_t* locale) { |
||||||
|
/* Handle byte comparison case. */ |
||||||
|
if (locale->nrules == 0) { |
||||||
|
size_t srclen = strlen(src); |
||||||
|
|
||||||
|
if (n != 0) { |
||||||
|
strncpy(dest, src, MIN(srclen + 1, n)); |
||||||
|
} |
||||||
|
|
||||||
|
return srclen; |
||||||
|
} |
||||||
|
|
||||||
|
/* Handle an empty string, code hereafter relies on strlen (src) > 0. */ |
||||||
|
if (*src == L('\0')) { |
||||||
|
if (n != 0) |
||||||
|
*dest = L('\0'); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* We need the elements of the string as unsigned values since they
|
||||||
|
are used as indeces. */ |
||||||
|
const USTRING_TYPE* usrc = (const USTRING_TYPE*)src; |
||||||
|
|
||||||
|
/* Allocate cache for small strings on the stack and fill it with weight and
|
||||||
|
rule indices. If the cache size is not sufficient, continue with the |
||||||
|
uncached xfrm version. */ |
||||||
|
size_t idxmax = 0; |
||||||
|
const USTRING_TYPE* cur = usrc; |
||||||
|
int32_t* idxarr = alloca(SMALL_STR_SIZE * sizeof(int32_t)); |
||||||
|
unsigned char* rulearr = alloca(SMALL_STR_SIZE + 1); |
||||||
|
|
||||||
|
do { |
||||||
|
int32_t tmp = |
||||||
|
findidx(locale->table, locale->indirect, locale->extra, &cur, -1); |
||||||
|
rulearr[idxmax] = tmp >> 24; |
||||||
|
idxarr[idxmax] = tmp & 0xffffff; |
||||||
|
|
||||||
|
++idxmax; |
||||||
|
} while (*cur != L('\0') && idxmax < SMALL_STR_SIZE); |
||||||
|
|
||||||
|
/* This element is only read, the value never used but to determine
|
||||||
|
another value which then is ignored. */ |
||||||
|
rulearr[idxmax] = '\0'; |
||||||
|
|
||||||
|
/* Do the transformation. */ |
||||||
|
if (*cur == L('\0')) |
||||||
|
return do_xfrm_cached(dest, n, locale, idxmax, idxarr, rulearr); |
||||||
|
else |
||||||
|
return do_xfrm(usrc, dest, n, locale); |
||||||
|
} |
||||||
Loading…
Reference in new issue