From 6b2b43d2833b74d690c172e39384a1713dc27c53 Mon Sep 17 00:00:00 2001 From: Alexandre Erwin Ittner Date: Sun, 18 Sep 2005 17:46:53 +0000 Subject: [PATCH] *** empty log message *** git-svn-id: file:///var/svn/lua-iconv/trunk@18 9538949d-8f27-0410-946f-ce01ef448559 --- Makefile | 35 ++++++++----- README | 135 +++++++++++++++++++++++++++++++++++++++++++++++++ fp.txt | 16 ++++++ fp.utf16.txt | Bin 0 -> 536 bytes fp.utf32.txt | Bin 0 -> 1072 bytes fp.utf8.txt | 16 ++++++ install51.lua | 92 +++++++++++++++++++++++++++++++++ luaiconv.c | 11 +++- makestr.lua | 18 +++++++ test_iconv.lua | 112 +++++++++++++++++++++++++++++++++++----- 10 files changed, 409 insertions(+), 26 deletions(-) create mode 100644 fp.txt create mode 100644 fp.utf16.txt create mode 100644 fp.utf32.txt create mode 100644 fp.utf8.txt create mode 100644 install51.lua create mode 100644 makestr.lua diff --git a/Makefile b/Makefile index a7c1356..4664305 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,28 @@ # luaiconv - Performs character set conversions in Lua # (c) 2005 Alexandre Erwin Ittner # +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. # -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHOR OR COPYRIGHT HOLDER BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # +# If you use this package in a product, an acknowledgment in the product +# documentation would be greatly appreciated (but it is not required). + CC=gcc OUTFILE=libluaiconv.so @@ -33,5 +39,8 @@ $(OUTFILE): luaiconv.c install: $(OUTFILE) cp $(OUTFILE) /usr/lib/ +install51: $(OUTFILE) + lua install51.lua $(OUTFILE) iconv + clean: rm -f $(OUTFILE) *.o diff --git a/README b/README index e69de29..61e2780 100644 --- a/README +++ b/README @@ -0,0 +1,135 @@ + + Lua-iconv: performs character set conversions in Lua + (c) 2005 Alexandre Erwin Ittner + Project page: http://lua-iconv.luaforge.net/ + + + + +=== INTRODUCTION === + +Lua-iconv is a Lua binding to the POSIX 'iconv' library. The iconv +library converts the sequence of characters from one codeset, into a +sequence of corresponding characters in another codeset. The codesets +are those specified in the iconv.new() call that returned the conversion +descriptor, cd. + +Details regarding iconv may be obtained from: +http://www.opengroup.org/onlinepubs/007908799/xsh/iconv.h.html + + + +=== LICENSE === + +Lua-iconv is (c) 2005 Alexandre Erwin Ittner + +Lua-iconv is copyrighted free software: it can be used for both academic +and commercial purposes at absolutely no cost. There are no royalties +or GNU-like "copyleft" restrictions. The legal details are below: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject + to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHOR OR COPYRIGHT HOLDER BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + If you use this package in a product, an acknowledgment in the product + documentation would be greatly appreciated (but it is not required). + + + +=== DOWNLOAD AND INSTALLATION === + +Lua-iconv can be obtained from its Luaforge project page: +http://lua-iconv.luaforge.net/ + +Lua-iconv must be compiled as a shared object and loaded by the Lua +interpreter using the 'loadlib' facility (or the new module system in +Lua 5.1). To compile the library, just fire up your favourite shell, untar +the distribution package and type, as root, within the program directory: + + make install + +The library will be compiled and installed on the /usr/lib/ directory. + +To install the library with Lua 5.1 and above just type + + make install51 + +This will cause the Makefile to run a Lua script that will install the +library into a directory from package.cpath, allowing you to use the +package system to load the library. + + + +=== LOADING AND INITIALIZATION === + +Lua-iconv is a shared library that exports a single function called +luaopen_iconv that must be called by the Lua interpreter. You must +use loadlib function to for that. Using Lua 5.0, under Unix and with +libluaiconv.so on your LUA_PATH (or also your LD_PATH) you can just do: + + assert(loadlib("libluaiconv.so", "luaopen_iconv"))() + +And, under Windows and with libluaiconv.dll on your path, you can do + + assert(loadlib("libluaiconv.dll", "luaopen_iconv"))() + +On the Lua 5.1, Lua-iconv uses the new package system that allows you +to simply do a + + require "gd" + +call to load up the library (that was installed on the right place by +install51.lua). + + + +=== USAGE === + +cd = iconv.new(to, from) +cd = iconv.open(to, from) + + Opens a new conversion descriptor, from the 'from' charset to the + 'to' charset. Concatenating "//TRANSLIT" to the first argument will + enable character transliteration and concatenating "//IGNORE" to + the first argument will cause iconv to ignore any invalid characters + found in the input string. + + This function returns a new converter or nil on error. + + +nstr, err = cd:iconv(str) + + Converts the 'str' string to the desired charset. This method always + returns two arguments: the converted string and an error code, which + may have any of the following values: + + nil + No error. Conversion was succeful. + + iconv.ERROR_NO_MEMORY + Failed to allocate enough memory in the conversion process. + + iconv.ERROR_INVALID + An invalid character was found in the input sequence. + + iconv.ERROR_INCOMPLETE + An incomplete character was found in the input sequence. + + iconv.ERROR_UNKNOWN + There was an unknown error. + diff --git a/fp.txt b/fp.txt new file mode 100644 index 0000000..f48bb68 --- /dev/null +++ b/fp.txt @@ -0,0 +1,16 @@ +Ao longe, ao luar +No rio uma vela +Serena a passar, +Que é que me revela + +Não sei, mas mue ser +Tornou-se-me estranho, +E eu sonho sem ver +Os sonhos que tenho. + +Que angústia me enlaça? +Que amor não se explica? +É a vela que passa +Na noite que fica + + -- Fernando Pessoa diff --git a/fp.utf16.txt b/fp.utf16.txt new file mode 100644 index 0000000000000000000000000000000000000000..f8cb0001285e3591893704a68b8e76c63d523a7c GIT binary patch literal 536 zcmY+BOHRW;5JYS3Q_KMpu7I+DB_F{BA{c@te_$M=T#<8R5pe|;z^m~@HnPU zF8}^))GMgr-D#(&6}N4{MxvxjH~F4a=kKT%Jm>>-sJNHI3@VI^j<6a~A zpfApQiQA00Ztmn+$2?^Kud^CY8>nkboikST2rn_S2VcX8yr-U#+n(H(>??UIxS~UZ z8OV88$)we}Ew z>_hWCV@1Dudo;%pSj+f=uVx>yz29D*iFW3RyY%wUgofqHYgHcm~ik6)d2+-C;q^8lZ*o{e(k zY(8ImG;d#ThrM?nz0|Ndf literal 0 HcmV?d00001 diff --git a/fp.utf8.txt b/fp.utf8.txt new file mode 100644 index 0000000..3646098 --- /dev/null +++ b/fp.utf8.txt @@ -0,0 +1,16 @@ +Ao longe, ao luar +No rio uma vela +Serena a passar, +Que é que me revela + +Não sei, mas mue ser +Tornou-se-me estranho, +E eu sonho sem ver +Os sonhos que tenho. + +Que angústia me enlaça? +Que amor não se explica? +É a vela que passa +Na noite que fica + + -- Fernando Pessoa diff --git a/install51.lua b/install51.lua new file mode 100644 index 0000000..5440fbe --- /dev/null +++ b/install51.lua @@ -0,0 +1,92 @@ +#!/usr/bin/env lua +-- Easy way to put a library on the path. For Lua >= 5.1 only. +-- (c) 2005 Alexandre Erwin Ittner < aittner{at}netuno.com.br > +-- +-- This file is distributed under the same license that Lua 5.1 itself. +-- +-- $Id$ +-- + +-- Splits a package.cpath into a table. +function splitpath(str) + local t = {} + local v + for v in string.gfind(str, "([^;]+);") do + table.insert(t, v) + end + return t +end + + +-- Checks if a given file exists. DUMB. Do not distinguish files from +-- symlinks, pipes, sockets, device drivers, etc. +function fileexists(fname) + local fp = io.open(fname, "r") + if fp then + fp:close() + return true + end + return false +end + + +-- Finds the first absolute path on the list. Works with *nix (MacOS X +-- included) and Windows paths. +function findplace(tbl) + local k, v + for k, v in ipairs(tbl) do + if v:sub(1, 1) == "/" + or v:sub(2, 3) == ":\\" then + return v + end + end +end + + +-- It's REALLY stupid, but also it's REALLY cross-platform. +function dumbcopy(from, to) + local bufsize = 512*1024 -- 512K at once + local ifp = io.open(from, "rb") + assert(ifp, "Can't open file for reading. File not found?") + local ofp = io.open(to, "wb") + assert(ofp, "Can't open file for writing. Permission denied?") + local buf + repeat + buf = ifp:read(bufsize) + if buf then + ofp:write(buf) + end + until not buf + ifp:close() + ofp:close() +end + + +-- Copies the file. 'to' MUST be an absolute path! +function copy(from, to) + if to:sub(1, 1) == "/" then -- Cool! We are on Unix, guys! + ret = os.execute('install -D "' .. from .. '" "' .. to .. '"') + if ret == 0 and fileexists(to) then + return + end + end + -- Let's try some violence. + dumbcopy(from, to) +end + + +-- Installs one file. Simple. Perfect. +function installfile(fname, name) + local tpath = splitpath(package.cpath) + local path = findplace(tpath) + assert(path, "Can't find a place to install") + local dest = path:gsub("%?", name) + print("Install:", dest) + copy(fname, dest) +end + +assert(package, "No package module found. Are you REALLY using Lua >= 5.1?") +assert(arg[1] and arg[2], "No arguments provided") + +installfile(arg[1], arg[2]) +os.exit(0) diff --git a/luaiconv.c b/luaiconv.c index 98fbcda..594da31 100644 --- a/luaiconv.c +++ b/luaiconv.c @@ -16,11 +16,13 @@ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * IN NO EVENT SHALL THE AUTHOR OR COPYRIGHT HOLDER BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + * If you use this package in a product, an acknowledgment in the product + * documentation would be greatly appreciated (but it is not required). * * $Id$ * @@ -35,8 +37,10 @@ #include #define LIB_NAME "iconv" +#define LIB_VERSION LIB_NAME " r1" #define ICONV_TYPENAME "iconv_t" + /* Compatibility between Lua 5.1+ and Lua 5.0 */ #ifndef LUA_VERSION_NUM #define LUA_VERSION_NUM 0 @@ -244,11 +248,16 @@ int luaopen_iconv(lua_State *L) tblseticons(L, "ERROR_INCOMPLETE", ERROR_INCOMPLETE); tblseticons(L, "ERROR_UNKNOWN", ERROR_UNKNOWN); + lua_pushliteral(L, "VERSION"); + lua_pushstring(L, LIB_VERSION); + lua_settable(L, -3); + luaL_newmetatable(L, ICONV_TYPENAME); lua_pushliteral(L, "__index"); lua_pushvalue(L, -3); lua_settable(L, -3); luaL_openlib(L, NULL, iconvMT, 0); + lua_pop(L, 1); return 1; } diff --git a/makestr.lua b/makestr.lua new file mode 100644 index 0000000..cb53d42 --- /dev/null +++ b/makestr.lua @@ -0,0 +1,18 @@ + +local fp = assert(io.open(arg[1], "rb")) +local str = assert(fp:read("*a")) +fp:close() + +local i +local c = 0 +local ostr = "local xxxxx = \"" +for i = 1, string.len(str) do + ostr = ostr .. "\\" .. string.byte(string.sub(str, i, i+1)) + if string.len(ostr) > 74 then + io.write(ostr .. "\"\n") + ostr = ".. \"" + end +end +io.write(ostr .. "\"\n") + + diff --git a/test_iconv.lua b/test_iconv.lua index 689cc79..55e6f71 100644 --- a/test_iconv.lua +++ b/test_iconv.lua @@ -1,19 +1,107 @@ -assert(loadlib("./libluaiconv.so", "luaopen_iconv"))() +require "iconv" -cd = iconv.new("utf-8", "iso-8859-1") ---cd = iconv.new("us-ascii//IGNORE", "iso-8859-1") ---cd = iconv.new("us-ascii//TRANSLIT", "iso-8859-1") +-- WARNING! This code assumes an iso-8859-1 terminal! -assert(cd, "Invalid conversion") +local iso88591 = [[ +Ao longe, ao luar +No rio uma vela +Serena a passar, +Que é que me revela ---ret, err = cd:iconv("Isso é um teste com acentuação") -ret, err = cd:iconv("ÃÁÉÍÓÚÇÑÝÔÕ") +Não sei, mas mue ser +Tornou-se-me estranho, +E eu sonho sem ver +Os sonhos que tenho. -if err == iconv.ERROR_INCOMPLETE then - print("Error: Incomplete input.") -elseif err == iconv.ERROR_INVALID then - print("Error: Invalid input.") +Que angústia me enlaça? +Que amor não se explica? +É a vela que passa +Na noite que fica + + -- Fernando Pessoa]] + + +local utf8 = "\65\111\32\108\111\110\103\101\44\32\97\111\32\108\117\97\114" +.. "\10\78\111\32\114\105\111\32\117\109\97\32\118\101\108\97\10\83\101\114" +.. "\101\110\97\32\97\32\112\97\115\115\97\114\44\10\81\117\101\32\195\169\32" +.. "\113\117\101\32\109\101\32\114\101\118\101\108\97\10\10\78\195\163\111\32" +.. "\115\101\105\44\32\109\97\115\32\109\117\101\32\115\101\114\10\84\111\114" +.. "\110\111\117\45\115\101\45\109\101\32\101\115\116\114\97\110\104\111\44" +.. "\10\69\32\101\117\32\115\111\110\104\111\32\115\101\109\32\118\101\114\10" +.. "\79\115\32\115\111\110\104\111\115\32\113\117\101\32\116\101\110\104\111" +.. "\46\10\10\81\117\101\32\97\110\103\195\186\115\116\105\97\32\109\101\32" +.. "\101\110\108\97\195\167\97\63\10\81\117\101\32\97\109\111\114\32\110\195" +.. "\163\111\32\115\101\32\101\120\112\108\105\99\97\63\10\195\137\32\97\32" +.. "\118\101\108\97\32\113\117\101\32\112\97\115\115\97\10\78\97\32\110\111" +.. "\105\116\101\32\113\117\101\32\102\105\99\97\10\10\32\32\32\32\45\45\32" +.. "\70\101\114\110\97\110\100\111\32\80\101\115\115\111\97\10" + +local utf16 = "\255\254\65\0\111\0\32\0\108\0\111\0\110\0\103\0\101\0\44\0\32" +.. "\0\97\0\111\0\32\0\108\0\117\0\97\0\114\0\10\0\78\0\111\0\32\0\114\0\105" +.. "\0\111\0\32\0\117\0\109\0\97\0\32\0\118\0\101\0\108\0\97\0\10\0\83\0\101" +.. "\0\114\0\101\0\110\0\97\0\32\0\97\0\32\0\112\0\97\0\115\0\115\0\97\0\114" +.. "\0\44\0\10\0\81\0\117\0\101\0\32\0\233\0\32\0\113\0\117\0\101\0\32\0\109" +.. "\0\101\0\32\0\114\0\101\0\118\0\101\0\108\0\97\0\10\0\10\0\78\0\227\0\111" +.. "\0\32\0\115\0\101\0\105\0\44\0\32\0\109\0\97\0\115\0\32\0\109\0\117\0\101" +.. "\0\32\0\115\0\101\0\114\0\10\0\84\0\111\0\114\0\110\0\111\0\117\0\45\0\115" +.. "\0\101\0\45\0\109\0\101\0\32\0\101\0\115\0\116\0\114\0\97\0\110\0\104\0" +.. "\111\0\44\0\10\0\69\0\32\0\101\0\117\0\32\0\115\0\111\0\110\0\104\0\111" +.. "\0\32\0\115\0\101\0\109\0\32\0\118\0\101\0\114\0\10\0\79\0\115\0\32\0\115" +.. "\0\111\0\110\0\104\0\111\0\115\0\32\0\113\0\117\0\101\0\32\0\116\0\101\0" +.. "\110\0\104\0\111\0\46\0\10\0\10\0\81\0\117\0\101\0\32\0\97\0\110\0\103\0" +.. "\250\0\115\0\116\0\105\0\97\0\32\0\109\0\101\0\32\0\101\0\110\0\108\0\97" +.. "\0\231\0\97\0\63\0\10\0\81\0\117\0\101\0\32\0\97\0\109\0\111\0\114\0\32" +.. "\0\110\0\227\0\111\0\32\0\115\0\101\0\32\0\101\0\120\0\112\0\108\0\105\0" +.. "\99\0\97\0\63\0\10\0\201\0\32\0\97\0\32\0\118\0\101\0\108\0\97\0\32\0\113" +.. "\0\117\0\101\0\32\0\112\0\97\0\115\0\115\0\97\0\10\0\78\0\97\0\32\0\110" +.. "\0\111\0\105\0\116\0\101\0\32\0\113\0\117\0\101\0\32\0\102\0\105\0\99\0" +.. "\97\0\10\0\10\0\32\0\32\0\32\0\32\0\45\0\45\0\32\0\70\0\101\0\114\0\110" +.. "\0\97\0\110\0\100\0\111\0\32\0\80\0\101\0\115\0\115\0\111\0\97\0\10\0" + + +-- Bizarre EBCDIC-CP-ES encoding. +local ebcdic = "\193\150\64\147\150\149\135\133\107\64\129\150\64\147\164\129" +.. "\153\37\213\150\64\153\137\150\64\164\148\129\64\165\133\147\129\37\226" +.. "\133\153\133\149\129\64\129\64\151\129\162\162\129\153\107\37\216\164\133" +.. "\64\81\64\152\164\133\64\148\133\64\153\133\165\133\147\129\37\37\213\70" +.. "\150\64\162\133\137\107\64\148\129\162\64\148\164\133\64\162\133\153\37" +.. "\227\150\153\149\150\164\96\162\133\96\148\133\64\133\162\163\153\129\149" +.. "\136\150\107\37\197\64\133\164\64\162\150\149\136\150\64\162\133\148\64" +.. "\165\133\153\37\214\162\64\162\150\149\136\150\162\64\152\164\133\64\163" +.. "\133\149\136\150\75\37\37\216\164\133\64\129\149\135\222\162\163\137\129" +.. "\64\148\133\64\133\149\147\129\72\129\111\37\216\164\133\64\129\148\150" +.. "\153\64\149\70\150\64\162\133\64\133\167\151\147\137\131\129\111\37\113" +.. "\64\129\64\165\133\147\129\64\152\164\133\64\151\129\162\162\129\37\213" +.. "\129\64\149\150\137\163\133\64\152\164\133\64\134\137\131\129\37\37\64\64" +.. "\64\64\96\96\64\198\133\153\149\129\149\132\150\64\215\133\162\162\150\129" +.. "\37" + + +function check_one(to, from, text) + print("\n-- Testing conversion from " .. from .. " to " .. to) + local cd = iconv.new(to .. "//TRANSLIT", from) + if not cd then + printf("FAILED!") + return + end + local ostr, err = cd:iconv(text) + + if err == iconv.ERROR_INCOMPLETE then + print("ERROR: Incomplete input.") + elseif err == iconv.ERROR_INVALID then + print("ERROR: Invalid input.") + elseif err == iconv.ERROR_NO_MEMORY then + print("ERROR: Failed to allocate memory.") + elseif err == iconv.ERROR_UNKNOWN then + print("ERROR: There was an unknown error.") + end + + print(ostr) end -print("Result: ", ret) +check_one("us-ascii", "iso8859-1", iso88591) +check_one("iso8859-1", "utf8", utf8) +check_one("iso8859-1", "utf16", utf16) +check_one("iso8859-1", "EBCDIC-CP-ES", ebcdic) +