encoding error in python 27

[Hala Gamal]

my code works well with english file but when i use text file encodede"utf-8" "my file contain some arabic letters" it doesn't work.
my code:
# encoding: utf-8
from whoosh import fields, index
import os.path
import re,string
import codecs
from whoosh.qparser import QueryParser

# This list associates a name with each position in a row
columns = ["juza","chapter","verse","voc"]

schema = fields.Schema(juza=fields.NUMERIC(stored=True),
chapter=fields.NUMERIC(stored=True),
verse=fields.NUMERIC(stored=True),
voc=fields.TEXT(stored=True))

# Create the Whoosh index
indexname = "indexdir"
if not os.path.exists(indexname):
os.mkdir(indexname)
ix = index.create_in(indexname, schema)

# Open a writer for the index
with ix.writer() as writer:
with codecs.open("tt.txt",encoding='utf-8') as txtfile:
lines=txtfile.readlines()

# Read each row in the file
for i in lines:

# Create a dictionary to hold the document values for this row
doc = {}
thisline=i.split()
u=0

# Read the values for the row enumerated like
# (0, "juza"), (1, "chapter"), etc.
for w in thisline:
# Get the field name from the "columns" list
fieldname = columns
u+=1
#if isinstance(w, basestring):
#w = unicode(w)
doc[fieldname] = w
# Pass the dictionary to the add_document method
writer.add_document(**doc)
with ix.searcher() as searcher:
query = QueryParser("voc", ix.schema).parse(u"كتاب")
results = searcher.search(query)
print(len(results))
print(results[1])
my error:
Traceback (most recent call last):
File "D:\Python27\yarab (4).py", line 45, in <module>
writer.add_document(**doc)
File "build\bdist.win32\egg\whoosh\filedb\filewriting.py", line 369, in add_document
items = field.index(value)
File "build\bdist.win32\egg\whoosh\fields.py", line 466, in index
return [(txt, 1, 1.0, '') for txt in self._tiers(num)]
File "build\bdist.win32\egg\whoosh\fields.py", line 454, in _tiers
yield self.to_text(num, shift=shift)
File "build\bdist.win32\egg\whoosh\fields.py", line 487, in to_text
return self._to_text(self.prepare_number(x), shift=shift,
File "build\bdist.win32\egg\whoosh\fields.py", line 476, in prepare_number
x = self.type(x)
UnicodeEncodeError: 'decimal' codec can't encode character u'\ufeff' in position 0: invalid decimal Unicode string
**my file:
2 2 3 كتاب
2 2 1 لعبة
1 1 1 كتاب
**any help?

 

[Peter Otten]

my code works well with english file but when i use text file
encodede"utf-8" "my file contain some arabic letters" it doesn't work. my
code:

with codecs.open("tt.txt",encoding='utf-8') as txtfile:

Try encoding="utf-8-sig" in the above to remove the byte order mark (BOM)
upon decoding, see

http://docs.python.org/2.7/library/codecs.html#module-encodings.utf_8_sig

That should prevent

 

[MRAB]

my code works well with english file but when i use text file encodede"utf-8" "my file contain some arabic letters" it doesn't work.
my code:
# encoding: utf-8
from whoosh import fields, index
import os.path
import re,string
import codecs
from whoosh.qparser import QueryParser

# This list associates a name with each position in a row
columns = ["juza","chapter","verse","voc"]

schema = fields.Schema(juza=fields.NUMERIC(stored=True),
chapter=fields.NUMERIC(stored=True),
verse=fields.NUMERIC(stored=True),
voc=fields.TEXT(stored=True))

# Create the Whoosh index
indexname = "indexdir"
if not os.path.exists(indexname):
os.mkdir(indexname)
ix = index.create_in(indexname, schema)

# Open a writer for the index
with ix.writer() as writer:
with codecs.open("tt.txt",encoding='utf-8') as txtfile:
lines=txtfile.readlines()

# Read each row in the file
for i in lines:

# Create a dictionary to hold the document values for this row
doc = {}
thisline=i.split()
u=0

# Read the values for the row enumerated like
# (0, "juza"), (1, "chapter"), etc.
for w in thisline:
# Get the field name from the "columns" list
fieldname = columns
u+=1
#if isinstance(w, basestring):
#w = unicode(w)
doc[fieldname] = w
# Pass the dictionary to the add_document method
writer.add_document(**doc)
with ix.searcher() as searcher:
query = QueryParser("voc", ix.schema).parse(u"كتاب")
results = searcher.search(query)
print(len(results))
print(results[1])
my error:
Traceback (most recent call last):
File "D:\Python27\yarab (4).py", line 45, in <module>
writer.add_document(**doc)
File "build\bdist.win32\egg\whoosh\filedb\filewriting.py", line 369, in add_document
items = field.index(value)
File "build\bdist.win32\egg\whoosh\fields.py", line 466, in index
return [(txt, 1, 1.0, '') for txt in self._tiers(num)]
File "build\bdist.win32\egg\whoosh\fields.py", line 454, in _tiers
yield self.to_text(num, shift=shift)
File "build\bdist.win32\egg\whoosh\fields.py", line 487, in to_text
return self._to_text(self.prepare_number(x), shift=shift,
File "build\bdist.win32\egg\whoosh\fields.py", line 476, in prepare_number
x = self.type(x)
UnicodeEncodeError: 'decimal' codec can't encode character u'\ufeff' in position 0: invalid decimal Unicode string
**my file:
2 2 3 كتاب
2 2 1 لعبة
1 1 1 كتاب
**any help?

I see that you're using Microsoft Windows.

Microsoft likes to indicate that a text file contains UTF-8 by starting
the text with u"\xFEFF" encoded as UTF-8. You're opening the file with
the encoding "utf-8", so you're seeing that marker.

Try opening the file with the encoding "utf-8-sig". That will drop the
marker if it's present.

 

[Hala Gamal]

thank you it worked well for small file but when i enter big file,, i obtain this error:
"Traceback (most recent call last):
File "D:\Python27\yarab (4).py", line 46, in <module>
writer.add_document(**doc)
File "build\bdist.win32\egg\whoosh\filedb\filewriting.py", line 369, in add_document
items = field.index(value)
File "build\bdist.win32\egg\whoosh\fields.py", line 466, in index
return [(txt, 1, 1.0, '') for txt in self._tiers(num)]
File "build\bdist.win32\egg\whoosh\fields.py", line 454, in _tiers
yield self.to_text(num, shift=shift)
File "build\bdist.win32\egg\whoosh\fields.py", line 487, in to_text
return self._to_text(self.prepare_number(x), shift=shift,
File "build\bdist.win32\egg\whoosh\fields.py", line 476, in prepare_number
x = self.type(x)
UnicodeEncodeError: 'decimal' codec can't encode characters in position 0-4: invalid decimal Unicode string"
i don't know realy where is the problem?

my code works well with english file but when i use text file encodede"utf-8" "my file contain some arabic letters" it doesn't work.

my code:

# encoding: utf-8

from whoosh import fields, index

import os.path

import re,string

import codecs

from whoosh.qparser import QueryParser



# This list associates a name with each position in a row

columns = ["juza","chapter","verse","voc"]



schema = fields.Schema(juza=fields.NUMERIC(stored=True),

chapter=fields.NUMERIC(stored=True),

verse=fields.NUMERIC(stored=True),

voc=fields.TEXT(stored=True))



# Create the Whoosh index

indexname = "indexdir"

if not os.path.exists(indexname):

os.mkdir(indexname)

ix = index.create_in(indexname, schema)



# Open a writer for the index

with ix.writer() as writer:

with codecs.open("tt.txt",encoding='utf-8') as txtfile:

lines=txtfile.readlines()



# Read each row in the file

for i in lines:



# Create a dictionary to hold the document values for this row

doc = {}

thisline=i.split()

u=0



# Read the values for the row enumerated like

# (0, "juza"), (1, "chapter"), etc.

for w in thisline:

# Get the field name from the "columns" list

fieldname = columns

u+=1

#if isinstance(w, basestring):

#w = unicode(w)

doc[fieldname] = w

# Pass the dictionary to the add_document method

writer.add_document(**doc)

with ix.searcher() as searcher:

query = QueryParser("voc", ix.schema).parse(u"كتاب")

results = searcher.search(query)

print(len(results))

print(results[1])

my error:

Traceback (most recent call last):

File "D:\Python27\yarab (4).py", line 45, in <module>

writer.add_document(**doc)

File "build\bdist.win32\egg\whoosh\filedb\filewriting.py", line 369, inadd_document

items = field.index(value)

File "build\bdist.win32\egg\whoosh\fields.py", line 466, in index

return [(txt, 1, 1.0, '') for txt in self._tiers(num)]

File "build\bdist.win32\egg\whoosh\fields.py", line 454, in _tiers

yield self.to_text(num, shift=shift)

File "build\bdist.win32\egg\whoosh\fields.py", line 487, in to_text

return self._to_text(self.prepare_number(x), shift=shift,

File "build\bdist.win32\egg\whoosh\fields.py", line 476, in prepare_number

x = self.type(x)

UnicodeEncodeError: 'decimal' codec can't encode character u'\ufeff' in position 0: invalid decimal Unicode string

**my file:

2 2 3 كتاب

2 2 1 لعبة

1 1 1 كتاب

**any help?

 

[Peter Otten]

thank you it worked well for small file but when i enter big file,, i
obtain this error: "Traceback (most recent call last):
File "D:\Python27\yarab (4).py", line 46, in <module>
writer.add_document(**doc)
File "build\bdist.win32\egg\whoosh\filedb\filewriting.py", line 369, in
add_document
items = field.index(value)
File "build\bdist.win32\egg\whoosh\fields.py", line 466, in index
return [(txt, 1, 1.0, '') for txt in self._tiers(num)]
File "build\bdist.win32\egg\whoosh\fields.py", line 454, in _tiers
yield self.to_text(num, shift=shift)
File "build\bdist.win32\egg\whoosh\fields.py", line 487, in to_text
return self._to_text(self.prepare_number(x), shift=shift,
File "build\bdist.win32\egg\whoosh\fields.py", line 476, in
prepare_number
x = self.type(x)
UnicodeEncodeError: 'decimal' codec can't encode characters in position
0-4: invalid decimal Unicode string" i don't know realy where is the

my code works well with english file but when i use text file
encodede"utf-8" "my file contain some arabic letters" it doesn't work.

I guess that one of the fields you require to be NUMERIC contains non-digit
characters. Replace the line

with something similar to

try:
writer.add_document(**doc)
except UnicodeEncodeError:
print "Skipping malformed line", repr(i)

This will allow you to inspect the lines your script cannot handle and if
they are indeed "malformed" as I am guessing you can fix your input data.

i is a terrible name for a line in a file, btw. Also, you should avoid
readlines() which reads the whole file into memory and instead iterate over
the file object directly:

with codecs.open("tt.txt", encoding='utf-8-sig') as textfile:
for line in textfile: # no readlines(), can handle
# text files of arbitrary size
...