[sword-cvs] icu-sword/source/tools/toolutil ucm.c,NONE,1.1 ucm.h,NONE,1.1 ucmstate.c,NONE,1.1 Makefile.in,1.3,1.4 toolutil.c,1.3,1.4 toolutil.dsp,1.3,1.4 toolutil.h,1.3,1.4 toolutil.vcproj,1.1,1.2 ucbuf.c,1.5,1.6 ucbuf.h,1.4,1.5 uparse.c,1.3,1.4 uperf.cpp,1.1,1.2
sword@www.crosswire.org
sword@www.crosswire.org
Tue, 6 Apr 2004 03:11:02 -0700
Update of /cvs/core/icu-sword/source/tools/toolutil
In directory www:/tmp/cvs-serv8911/source/tools/toolutil
Modified Files:
Makefile.in toolutil.c toolutil.dsp toolutil.h toolutil.vcproj
ucbuf.c ucbuf.h uparse.c uperf.cpp
Added Files:
ucm.c ucm.h ucmstate.c
Log Message:
ICU 2.8 sync
--- NEW FILE: ucm.c ---
/*
*******************************************************************************
*
* Copyright (C) 2003, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: ucm.c
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2003jun20
* created by: Markus W. Scherer
*
* This file reads a .ucm file, stores its mappings and sorts them.
* It implements handling of Unicode conversion mappings from .ucm files
* for makeconv, canonucm, rptp2ucm, etc.
*
[...1138 lines suppressed...]
}
*end=0;
/* ignore empty and comment lines */
if(line[0]==0 || line[0]=='#') {
continue;
}
/* stop at the end of the mapping table */
if(0==uprv_strcmp(line, "END CHARMAP")) {
break;
}
isOK&=ucm_addMappingFromLine(ucm, line, forBase, baseStates);
}
if(!isOK) {
*pErrorCode=U_INVALID_TABLE_FORMAT;
}
}
--- NEW FILE: ucm.h ---
/*
*******************************************************************************
*
* Copyright (C) 2003, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: ucm.h
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2003jun20
* created by: Markus W. Scherer
*
* Definitions for the .ucm file parser and handler module ucm.c.
*/
#ifndef __UCM_H__
#define __UCM_H__
#include "unicode/utypes.h"
#include "ucnvmbcs.h"
#include "ucnv_ext.h"
#include "filestrm.h"
#include <stdio.h>
U_CDECL_BEGIN
/*
* Per-mapping data structure
*
* u if uLen==1: Unicode code point
* else index to uLen code points
* b if bLen<=4: up to 4 bytes
* else index to bLen bytes
* uLen number of code points
* bLen number of words containing left-justified bytes
* bIsMultipleChars indicates that the bytes contain more than one sequence
* according to the state table
* f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3)
* same values as in the source file after |
*/
typedef struct UCMapping {
UChar32 u;
union {
uint32_t index;
uint8_t bytes[4];
} b;
int8_t uLen, bLen, f, moveFlag;
} UCMapping;
enum {
UCM_FLAGS_INITIAL, /* no mappings parsed yet */
UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */
UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */
UCM_FLAGS_MIXED /* both implicit and explicit */
};
typedef struct UCMTable {
UCMapping *mappings;
int32_t mappingsCapacity, mappingsLength;
UChar32 *codePoints;
int32_t codePointsCapacity, codePointsLength;
uint8_t *bytes;
int32_t bytesCapacity, bytesLength;
/* index map for mapping by bytes first */
int32_t *reverseMap;
uint8_t unicodeMask;
int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */
UBool isSorted;
} UCMTable;
enum {
MBCS_STATE_FLAG_DIRECT=1,
MBCS_STATE_FLAG_SURROGATES,
MBCS_STATE_FLAG_READY=16
};
typedef struct UCMStates {
int32_t stateTable[MBCS_MAX_STATE_COUNT][256];
uint32_t stateFlags[MBCS_MAX_STATE_COUNT],
stateOffsetSum[MBCS_MAX_STATE_COUNT];
int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits;
int8_t conversionType, outputType;
} UCMStates;
typedef struct UCMFile {
UCMTable *base, *ext;
UCMStates states;
char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH];
} UCMFile;
/* simple accesses ---------------------------------------------------------- */
#define UCM_GET_CODE_POINTS(t, m) \
(((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u)
#define UCM_GET_BYTES(t, m) \
(((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.index)
/* APIs --------------------------------------------------------------------- */
U_CAPI UCMFile * U_EXPORT2
ucm_open(void);
U_CAPI void U_EXPORT2
ucm_close(UCMFile *ucm);
U_CAPI UBool U_EXPORT2
ucm_parseHeaderLine(UCMFile *ucm,
char *line, char **pKey, char **pValue);
/* @return -1 illegal bytes 0 suitable for base table 1 needs to go into extension table */
U_CAPI int32_t U_EXPORT2
ucm_mappingType(UCMStates *baseStates,
UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES]);
/* add a mapping to the base or extension table as appropriate */
U_CAPI UBool U_EXPORT2
ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES]);
U_CAPI UBool U_EXPORT2
ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates);
U_CAPI UCMTable * U_EXPORT2
ucm_openTable(void);
U_CAPI void U_EXPORT2
ucm_closeTable(UCMTable *table);
U_CAPI void U_EXPORT2
ucm_resetTable(UCMTable *table);
U_CAPI void U_EXPORT2
ucm_sortTable(UCMTable *t);
/**
* Read a table from a .ucm file, from after the CHARMAP line to
* including the END CHARMAP line.
*/
U_CAPI void U_EXPORT2
ucm_readTable(UCMFile *ucm, FileStream* convFile,
UBool forBase, UCMStates *baseStates,
UErrorCode *pErrorCode);
/**
* Check the validity of mappings against a base table's states;
* necessary for extension-only tables that were read before their base tables.
*/
U_CAPI UBool U_EXPORT2
ucm_checkValidity(UCMTable *ext, UCMStates *baseStates);
/**
* Check a base table against an extension table.
* Set the moveTarget!=NULL if it is possible to move mappings from the base.
* This is the case where base and extension tables are parsed from a single file
* (moveTarget==ext)
* or when delta file mappings are subtracted from a base table.
*
* When a base table cannot be modified because a delta file is parsed in makeconv,
* then set moveTarget=NULL.
*
* if(intersectBase) then mappings that exist in the base table but not in
* the extension table are moved to moveTarget instead of showing an error.
*
* Special mode:
* If intersectBase==2 for a DBCS extension table, then SBCS mappings are
* not moved out of the base unless their Unicode input requires it.
* This helps ucmkbase generate base tables for DBCS-only extension .cnv files.
*
* For both tables in the same file, the extension table is automatically
* built.
* For separate files, the extension file can use a complete mapping table,
* so that common mappings need not be stripped out manually.
*
*
* Sort both tables, and then for each mapping direction:
*
* If intersectBase is TRUE and the base table contains a mapping
* that does not exist in the extension table, then this mapping is moved
* to moveTarget.
*
* - otherwise -
*
* If the base table contains a mapping for which the input sequence is
* the same as the extension input, then
* - if the output is the same: remove the extension mapping
* - else: error
*
* If the base table contains a mapping for which the input sequence is
* a prefix of the extension input, then
* - if moveTarget!=NULL: move the base mapping to the moveTarget table
* - else: error
*
* @return FALSE in case of an irreparable error
*/
U_CAPI UBool U_EXPORT2
ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
UCMTable *moveTarget, UBool intersectBase);
U_CAPI void U_EXPORT2
ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode);
U_CAPI void U_EXPORT2
ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f);
U_CAPI void U_EXPORT2
ucm_addState(UCMStates *states, const char *s);
U_CAPI void U_EXPORT2
ucm_processStates(UCMStates *states);
U_CAPI int32_t U_EXPORT2
ucm_countChars(UCMStates *states,
const uint8_t *bytes, int32_t length);
U_CAPI int8_t U_EXPORT2
ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps);
U_CAPI UBool U_EXPORT2
ucm_parseMappingLine(UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES],
const char *line);
U_CAPI void U_EXPORT2
ucm_addMapping(UCMTable *table,
UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES]);
/* very makeconv-specific functions ----------------------------------------- */
/* finalize and optimize states after the toUnicode mappings are processed */
U_CAPI void U_EXPORT2
ucm_optimizeStates(UCMStates *states,
uint16_t **pUnicodeCodeUnits,
_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
UBool verbose);
/* moved here because it is used inside ucmstate.c */
U_CAPI int32_t U_EXPORT2
ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
uint32_t offset);
/* very rptp2ucm-specific functions ----------------------------------------- */
/*
* Input: Separate tables with mappings from/to Unicode,
* subchar and subchar1 (0 if none).
* All mappings must have flag 0.
*
* Output: fromUTable will contain the union of mappings with the correct
* precision flags, and be sorted.
*/
U_CAPI void U_EXPORT2
ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
const uint8_t *subchar, int32_t subcharLength,
uint8_t subchar1);
U_CAPI UBool U_EXPORT2
ucm_separateMappings(UCMFile *ucm, UBool isSISO);
U_CDECL_END
#endif
--- NEW FILE: ucmstate.c ---
/*
*******************************************************************************
*
* Copyright (C) 2003, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: ucmstate.c
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2003oct09
* created by: Markus W. Scherer
*
* This file handles ICU .ucm file state information as part of the ucm module.
* Most of this code used to be in makeconv.c.
*/
[...1003 lines suppressed...]
}
}
}
if(offset!=0) {
fprintf(stderr, "ucm error: byte sequence too short, ends in non-final state %hu\n", state);
return -1;
}
/*
* for SI/SO (like EBCDIC-stateful), multiple-character results
* must consist of only double-byte sequences
*/
if(count>1 && states->outputType==MBCS_OUTPUT_2_SISO && length!=2*count) {
fprintf(stderr, "ucm error: SI/SO (like EBCDIC-stateful) result with %d characters does not contain all DBCS\n", count);
return -1;
}
return count;
}
Index: Makefile.in
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/Makefile.in,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- Makefile.in 10 Sep 2003 02:43:00 -0000 1.3
+++ Makefile.in 6 Apr 2004 10:10:27 -0000 1.4
@@ -20,12 +20,14 @@
## Target information
+TARGET_STUBNAME=toolutil
+
ifneq ($(ENABLE_STATIC),)
-TARGET = $(LIBICU)toolutil$(ICULIBSUFFIX).a
+TARGET = $(LIBSICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(A)
endif
ifneq ($(ENABLE_SHARED),)
-SO_TARGET = $(LIBICU)toolutil$(ICULIBSUFFIX).$(SO)
+SO_TARGET = $(LIBICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(SO)
ALL_SO_TARGETS = $(SO_TARGET) $(MIDDLE_SO_TARGET) $(FINAL_SO_TARGET)
endif
@@ -38,7 +40,7 @@
CPPFLAGS += -I$(top_builddir)/common -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(top_srcdir)/tools/ctestfw $(LIBCPPFLAGS)
LIBS = $(LIBICUUC) $(DEFAULT_LIBS)
-OBJECTS = toolutil.o unewdata.o ucmpwrit.o uoptions.o uparse.o ucbuf.o uperf.o
+OBJECTS = toolutil.o unewdata.o ucm.o ucmstate.o ucmpwrit.o uoptions.o uparse.o ucbuf.o uperf.o
STATIC_OBJECTS = $(OBJECTS:.o=.$(STATIC_O))
Index: toolutil.c
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/toolutil.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- toolutil.c 10 Sep 2003 02:43:00 -0000 1.3
+++ toolutil.c 6 Apr 2004 10:10:27 -0000 1.4
@@ -26,6 +26,7 @@
# define NOMCX
# include <windows.h>
#endif
+#include <stdio.h>
#include "unicode/utypes.h"
#include "unicode/putil.h"
#include "cmemory.h"
@@ -58,18 +59,133 @@
U_CAPI const char * U_EXPORT2
findBasename(const char *filename) {
const char *basename=uprv_strrchr(filename, U_FILE_SEP_CHAR);
- if(basename!=NULL) {
- return basename+1;
- } else {
-#ifdef WIN32
+
+#if U_FILE_ALT_SEP_CHAR!=U_FILE_SEP_CHAR
+ if(basename==NULL) {
/* Use lenient matching on Windows, which can accept either \ or /
- This is useful for CygWin environments which has both
+ This is useful for environments like Win32+CygWin which have both.
*/
- basename=uprv_strrchr(filename, '/');
- if(basename!=NULL) {
- return basename+1;
- }
+ basename=uprv_strrchr(filename, U_FILE_ALT_SEP_CHAR);
+ }
#endif
+
+ if(basename!=NULL) {
+ return basename+1;
+ } else {
return filename;
}
+}
+
+/* tool memory helper ------------------------------------------------------- */
+
+struct UToolMemory {
+ char name[64];
+ int32_t capacity, maxCapacity, size, index;
+ void *array;
+ UAlignedMemory staticArray[1];
+};
+
+U_CAPI UToolMemory * U_EXPORT2
+utm_open(const char *name, int32_t initialCapacity, int32_t maxCapacity, int32_t size) {
+ UToolMemory *mem;
+
+ if(maxCapacity<initialCapacity) {
+ maxCapacity=initialCapacity;
+ }
+
+ mem=(UToolMemory *)uprv_malloc(sizeof(UToolMemory)+initialCapacity*size);
+ if(mem==NULL) {
+ fprintf(stderr, "error: %s - out of memory\n", name);
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+ mem->array=mem->staticArray;
+
+ uprv_strcpy(mem->name, name);
+ mem->capacity=initialCapacity;
+ mem->maxCapacity=maxCapacity;
+ mem->size=size;
+ mem->index=0;
+ return mem;
+}
+
+U_CAPI void U_EXPORT2
+utm_close(UToolMemory *mem) {
+ if(mem!=NULL) {
+ if(mem->array!=mem->staticArray) {
+ uprv_free(mem->array);
+ }
+ uprv_free(mem);
+ }
+}
+
+
+U_CAPI void * U_EXPORT2
+utm_getStart(UToolMemory *mem) {
+ return (char *)mem->array;
+}
+
+U_CAPI int32_t U_EXPORT2
+utm_countItems(UToolMemory *mem) {
+ return mem->index;
+}
+
+
+static UBool
+utm_hasCapacity(UToolMemory *mem, int32_t capacity) {
+ if(mem->capacity<capacity) {
+ int32_t newCapacity;
+
+ if(mem->maxCapacity<capacity) {
+ fprintf(stderr, "error: %s - trying to use more than maxCapacity=%ld units\n",
+ mem->name, (long)mem->maxCapacity);
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+
+ /* try to allocate a larger array */
+ if(capacity>=2*mem->capacity) {
+ newCapacity=capacity;
+ } else if(mem->capacity<=mem->maxCapacity/3) {
+ newCapacity=2*mem->capacity;
+ } else {
+ newCapacity=mem->maxCapacity;
+ }
+
+ if(mem->array==mem->staticArray) {
+ mem->array=uprv_malloc(newCapacity*mem->size);
+ if(mem->array!=NULL) {
+ uprv_memcpy(mem->array, mem->staticArray, mem->index*mem->size);
+ }
+ } else {
+ mem->array=uprv_realloc(mem->array, newCapacity*mem->size);
+ }
+
+ if(mem->array==NULL) {
+ fprintf(stderr, "error: %s - out of memory\n", mem->name);
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+ }
+
+ return TRUE;
+}
+
+U_CAPI void * U_EXPORT2
+utm_alloc(UToolMemory *mem) {
+ char *p=(char *)mem->array+mem->index*mem->size;
+ int32_t newIndex=mem->index+1;
+ if(utm_hasCapacity(mem, newIndex)) {
+ mem->index=newIndex;
+ uprv_memset(p, 0, mem->size);
+ }
+ return p;
+}
+
+U_CAPI void * U_EXPORT2
+utm_allocN(UToolMemory *mem, int32_t n) {
+ char *p=(char *)mem->array+mem->index*mem->size;
+ int32_t newIndex=mem->index+n;
+ if(utm_hasCapacity(mem, newIndex)) {
+ mem->index=newIndex;
+ uprv_memset(p, 0, n*mem->size);
+ }
+ return p;
}
Index: toolutil.dsp
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/toolutil.dsp,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- toolutil.dsp 10 Sep 2003 02:43:00 -0000 1.3
+++ toolutil.dsp 6 Apr 2004 10:10:27 -0000 1.4
@@ -56,7 +56,7 @@
# ADD BSC32 /nologo
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386
-# ADD LINK32 kernel32.lib /nologo /dll /machine:I386 /out:"..\..\..\bin\icutu26.dll" /implib:"..\..\..\lib/icutu.lib" /libpath:"..\..\..\lib"
+# ADD LINK32 kernel32.lib /nologo /dll /machine:I386 /out:"..\..\..\bin\icutu28.dll" /implib:"..\..\..\lib/icutu.lib" /libpath:"..\..\..\lib"
# SUBTRACT LINK32 /pdb:none
!ELSEIF "$(CFG)" == "toolutil - Win32 Debug"
@@ -84,7 +84,7 @@
# ADD BSC32 /nologo
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept
-# ADD LINK32 kernel32.lib /nologo /dll /debug /machine:I386 /out:"..\..\..\bin\icutu26d.dll" /implib:"..\..\..\lib/icutud.lib" /pdbtype:sept /libpath:"..\..\..\lib"
+# ADD LINK32 kernel32.lib /nologo /dll /debug /machine:I386 /out:"..\..\..\bin\icutu28d.dll" /implib:"..\..\..\lib/icutud.lib" /pdbtype:sept /libpath:"..\..\..\lib"
# SUBTRACT LINK32 /pdb:none
!ELSEIF "$(CFG)" == "toolutil - Win64 Release"
@@ -112,7 +112,7 @@
# ADD BSC32 /nologo
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:IX86 /machine:IA64
-# ADD LINK32 kernel32.lib /nologo /dll /machine:IX86 /out:"..\..\..\bin\icutu26.dll" /implib:"..\..\..\lib/icutu.lib" /libpath:"..\..\..\lib" /machine:IA64
+# ADD LINK32 kernel32.lib /nologo /dll /machine:IX86 /out:"..\..\..\bin\icutu28.dll" /implib:"..\..\..\lib/icutu.lib" /libpath:"..\..\..\lib" /machine:IA64
!ELSEIF "$(CFG)" == "toolutil - Win64 Debug"
@@ -139,7 +139,7 @@
# ADD BSC32 /nologo
LINK32=link.exe
# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:IX86 /pdbtype:sept /machine:IA64
-# ADD LINK32 kernel32.lib /nologo /dll /incremental:no /debug /machine:IX86 /out:"..\..\..\bin\icutu26d.dll" /implib:"..\..\..\lib/icutud.lib" /pdbtype:sept /libpath:"..\..\..\lib" /machine:IA64
+# ADD LINK32 kernel32.lib /nologo /dll /incremental:no /debug /machine:IX86 /out:"..\..\..\bin\icutu28d.dll" /implib:"..\..\..\lib/icutud.lib" /pdbtype:sept /libpath:"..\..\..\lib" /machine:IA64
!ENDIF
@@ -163,10 +163,18 @@
# End Source File
# Begin Source File
+SOURCE=.\ucm.c
+# End Source File
+# Begin Source File
+
SOURCE=.\ucmpwrit.c
# End Source File
# Begin Source File
+SOURCE=.\ucmstate.c
+# End Source File
+# Begin Source File
+
SOURCE=.\unewdata.c
# End Source File
# Begin Source File
@@ -192,6 +200,10 @@
# Begin Source File
SOURCE=.\ucbuf.h
+# End Source File
+# Begin Source File
+
+SOURCE=.\ucm.h
# End Source File
# Begin Source File
Index: toolutil.h
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/toolutil.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- toolutil.h 10 Sep 2003 02:43:00 -0000 1.3
+++ toolutil.h 6 Apr 2004 10:10:27 -0000 1.4
@@ -1,7 +1,7 @@
/*
*******************************************************************************
*
-* Copyright (C) 1999-2000, International Business Machines
+* Copyright (C) 1999-2003, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
@@ -21,8 +21,6 @@
#include "unicode/utypes.h"
-
-
/*
* For Windows, a path/filename may be the short (8.3) version
* of the "real", long one. In this case, the short one
@@ -50,5 +48,56 @@
*/
U_CAPI const char * U_EXPORT2
findBasename(const char *filename);
+
+/*
+ * UToolMemory is used for generic, custom memory management.
+ * It is allocated with enough space for count*size bytes starting
+ * at array.
+ * The array is declared with a union of large data types so
+ * that its base address is aligned for any types.
+ * If size is a multiple of a data type size, then such items
+ * can be safely allocated inside the array, at offsets that
+ * are themselves multiples of size.
+ */
+struct UToolMemory;
+typedef struct UToolMemory UToolMemory;
+
+/**
+ * Open a UToolMemory object for allocation of initialCapacity to maxCapacity
+ * items with size bytes each.
+ */
+U_CAPI UToolMemory * U_EXPORT2
+utm_open(const char *name, int32_t initialCapacity, int32_t maxCapacity, int32_t size);
+
+/**
+ * Close a UToolMemory object.
+ */
+U_CAPI void U_EXPORT2
+utm_close(UToolMemory *mem);
+
+/**
+ * Get the pointer to the beginning of the array of items.
+ * The pointer becomes invalid after allocation of new items.
+ */
+U_CAPI void * U_EXPORT2
+utm_getStart(UToolMemory *mem);
+
+/**
+ * Get the current number of items.
+ */
+U_CAPI int32_t U_EXPORT2
+utm_countItems(UToolMemory *mem);
+
+/**
+ * Allocate one more item and return the pointer to its start in the array.
+ */
+U_CAPI void * U_EXPORT2
+utm_alloc(UToolMemory *mem);
+
+/**
+ * Allocate n items and return the pointer to the start of the first one in the array.
+ */
+U_CAPI void * U_EXPORT2
+utm_allocN(UToolMemory *mem, int32_t n);
#endif
Index: toolutil.vcproj
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/toolutil.vcproj,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- toolutil.vcproj 10 Sep 2003 02:43:00 -0000 1.1
+++ toolutil.vcproj 6 Apr 2004 10:10:27 -0000 1.2
@@ -1,7 +1,7 @@
-<?xml version="1.0" encoding = "Windows-1252"?>
+<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
- Version="7.00"
+ Version="7.10"
Name="toolutil"
SccProjectName=""
SccLocalPath="">
@@ -39,11 +39,11 @@
<Tool
Name="VCLinkerTool"
AdditionalOptions="/MACHINE:I386"
- OutputFile="..\..\..\bin\icutu26.dll"
+ OutputFile="..\..\..\bin\icutu28.dll"
LinkIncremental="1"
SuppressStartupBanner="TRUE"
AdditionalLibraryDirectories="..\..\..\lib"
- ProgramDatabaseFile=".\..\..\..\lib/icutu26.pdb"
+ ProgramDatabaseFile=".\..\..\..\lib/icutu28.pdb"
ImportLibrary="..\..\..\lib/icutu.lib"/>
<Tool
Name="VCMIDLTool"
@@ -65,7 +65,13 @@
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
+ Name="VCXMLDataGeneratorTool"/>
+ <Tool
Name="VCWebDeploymentTool"/>
+ <Tool
+ Name="VCManagedWrapperGeneratorTool"/>
+ <Tool
+ Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
<Configuration
Name="Debug|Win32"
@@ -96,12 +102,12 @@
<Tool
Name="VCLinkerTool"
AdditionalOptions="/MACHINE:I386"
- OutputFile="..\..\..\bin\icutu26d.dll"
+ OutputFile="..\..\..\bin\icutu28d.dll"
LinkIncremental="2"
SuppressStartupBanner="TRUE"
AdditionalLibraryDirectories="..\..\..\lib"
GenerateDebugInformation="TRUE"
- ProgramDatabaseFile=".\..\..\..\lib/icutu26d.pdb"
+ ProgramDatabaseFile=".\..\..\..\lib/icutu28d.pdb"
ImportLibrary="..\..\..\lib/icutud.lib"/>
<Tool
Name="VCMIDLTool"
@@ -123,9 +129,17 @@
<Tool
Name="VCWebServiceProxyGeneratorTool"/>
<Tool
+ Name="VCXMLDataGeneratorTool"/>
+ <Tool
Name="VCWebDeploymentTool"/>
+ <Tool
+ Name="VCManagedWrapperGeneratorTool"/>
+ <Tool
+ Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
</Configuration>
</Configurations>
+ <References>
+ </References>
<Files>
<Filter
Name="Source Files"
@@ -137,9 +151,15 @@
RelativePath=".\ucbuf.c">
</File>
<File
+ RelativePath=".\ucm.c">
+ </File>
+ <File
RelativePath=".\ucmpwrit.c">
</File>
<File
+ RelativePath=".\ucmstate.c">
+ </File>
+ <File
RelativePath=".\unewdata.c">
</File>
<File
@@ -160,6 +180,9 @@
</File>
<File
RelativePath=".\ucbuf.h">
+ </File>
+ <File
+ RelativePath=".\ucm.h">
</File>
<File
RelativePath=".\ucmpwrit.h">
Index: ucbuf.c
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/ucbuf.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- ucbuf.c 10 Sep 2003 02:43:00 -0000 1.5
+++ ucbuf.c 6 Apr 2004 10:10:27 -0000 1.6
@@ -76,7 +76,7 @@
pTarget = target;
pStart = start;
ucnv_toUnicode(*conv, &pTarget, target+1, &pStart, start+*signatureLength, NULL, FALSE, error);
- *signatureLength = pStart - start;
+ *signatureLength = (int32_t)(pStart - start);
if(*error==U_BUFFER_OVERFLOW_ERROR) {
*error=U_ZERO_ERROR;
}
@@ -87,7 +87,6 @@
}
-
return TRUE;
}
static UBool ucbuf_isCPKnown(const char* cp){
@@ -112,13 +111,10 @@
if(ucnv_compareNames("UTF-32LE",cp)==0){
return TRUE;
}
- if(ucnv_compareNames("UTF-32BE",cp)==0){
- return TRUE;
- }
if(ucnv_compareNames("SCSU",cp)==0){
return TRUE;
}
- if(ucnv_compareNames("BOCU",cp)==0){
+ if(ucnv_compareNames("BOCU-1",cp)==0){
return TRUE;
}
if(ucnv_compareNames("UTF-7",cp)==0){
@@ -611,7 +607,7 @@
if(buf->isBuffered){
return (T_FileStream_size(buf->in)-buf->signatureLength)/ucnv_getMinCharSize(buf->conv);
}else{
- return buf->bufLimit-buf->buffer;
+ return (int32_t)(buf->bufLimit - buf->buffer);
}
}
return 0;
@@ -626,7 +622,7 @@
*error = U_ILLEGAL_ARGUMENT_ERROR;
return NULL;
}
- *len = buf->bufLimit-buf->buffer;
+ *len = (int32_t)(buf->bufLimit - buf->buffer);
return buf->buffer;
}
@@ -712,13 +708,13 @@
UChar* savePos =NULL;
UChar c=0x0000;
if(buf->isBuffered){
- /* The input is buffered we have to do more
- * for returning a pointer U_TRUNCATED_CHAR_FOUND
+ /* The input is buffered we have to do more
+ * for returning a pointer U_TRUNCATED_CHAR_FOUND
*/
for(;;){
c = *temp++;
if(buf->remaining==0){
- *err = (UErrorCode) U_EOF;
+ return NULL; /* end of file is reached return NULL */
}
if(temp>=buf->bufLimit && buf->currentPos == buf->buffer){
*err= U_TRUNCATED_CHAR_FOUND;
@@ -726,7 +722,7 @@
}else{
ucbuf_fillucbuf(buf,err);
if(U_FAILURE(*err)){
- return NULL;
+ return NULL;
}
}
/*
@@ -735,7 +731,7 @@
*/
/* Windows CR LF */
if(c ==0x0d && temp+1<=buf->bufLimit && *(temp+1) == 0x0a ){
- *len = temp++ - buf->currentPos;
+ *len = (int32_t)(temp++ - buf->currentPos);
savePos = buf->currentPos;
buf->currentPos = temp;
return savePos;
@@ -743,7 +739,7 @@
/* else */
if (temp>=buf->bufLimit|| ucbuf_isCharNewLine(c)){ /* Unipad inserts 2028 line separators! */
- *len = temp - buf->currentPos;
+ *len = (int32_t)(temp - buf->currentPos);
savePos = buf->currentPos;
buf->currentPos = temp;
return savePos;
@@ -757,19 +753,18 @@
c = *temp++;
if(buf->currentPos==buf->bufLimit){
- *err = (UErrorCode) U_EOF;
- return NULL;
+ return NULL; /* end of file is reached return NULL */
}
/* Windows CR LF */
if(c ==0x0d && temp+1<=buf->bufLimit && *(temp+1) == 0x0a ){
- *len = temp++ - buf->currentPos;
+ *len = (int32_t)(temp++ - buf->currentPos);
savePos = buf->currentPos;
buf->currentPos = temp;
return savePos;
}
/* else */
if (temp>=buf->bufLimit|| ucbuf_isCharNewLine(c)) { /* Unipad inserts 2028 line separators! */
- *len = temp - buf->currentPos;
+ *len = (int32_t)(temp - buf->currentPos);
savePos = buf->currentPos;
buf->currentPos = temp;
return savePos;
Index: ucbuf.h
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/ucbuf.h,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- ucbuf.h 10 Sep 2003 02:43:00 -0000 1.4
+++ ucbuf.h 6 Apr 2004 10:10:27 -0000 1.5
@@ -103,7 +103,7 @@
* indicates a failure on entry, the function will immediately return.
* On exit the value will indicate the success of the operation.
* Error: U_TRUNCATED_CHAR_FOUND
- * @return Pointer to the internal buffer
+ * @return Pointer to the internal buffer, NULL if EOF
*/
U_CAPI const UChar* U_EXPORT2
ucbuf_readline(UCHARBUF* buf,int32_t* len, UErrorCode* err);
Index: uparse.c
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/uparse.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- uparse.c 10 Sep 2003 02:43:00 -0000 1.3
+++ uparse.c 6 Apr 2004 10:10:28 -0000 1.4
@@ -385,7 +385,7 @@
int32_t i = 0;
unsigned int value = 0;
if(sLen == -1) {
- sLen = strlen(source);
+ sLen = (int32_t)strlen(source);
}
while(read < source+sLen) {
Index: uperf.cpp
===================================================================
RCS file: /cvs/core/icu-sword/source/tools/toolutil/uperf.cpp,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- uperf.cpp 10 Sep 2003 02:43:00 -0000 1.1
+++ uperf.cpp 6 Apr 2004 10:10:28 -0000 1.2
@@ -157,7 +157,7 @@
int32_t len =0;
for (;;) {
line = ucbuf_readline(ucharBuf,&len,&status);
- if(status == U_EOF||U_FAILURE(status)){
+ if(line == NULL || U_FAILURE(status)){
break;
}
lines[numLines].name = new UChar[len];
@@ -180,9 +180,6 @@
delete lines;
lines = newLines;
}
- }
- if(status==U_EOF){
- status =U_ZERO_ERROR;
}
return lines;
}