>>> s = "中文"
>>> s
'中文'
>>> type(s)
<class 'str'>
>>> b = bytes(s, encoding='utf-8')
>>> b
b'\xe4\xb8\xad\xe6\x96\x87'   #\x 代表是十六进制
>>> type(b)
<class 'bytes'>

>>> b
b'\xe4\xb8\xad\xe6\x96\x87'
>>> type(b)
<class 'bytes'>

>>> s1 = str(b)
>>> s1
"b'\\xe4\\xb8\\xad\\xe6\\x96\\x87'"  #注意这里
>>> type(s1)
<class 'str'>

>>> s1 = str(b, encoding='utf-8')
>>> s1
'中文'
>>> type(s1)
<class 'str'>

>>> s1
'中文'
>>> type(s1)
<class 'str'>
>>> b = bytes(s1, encoding='gbk')
>>> b

>>> a = 'hello world'
>>> type(a)
<class 'str'>
>>> a
'hello world'

#a为串'hello world'
>>> b = a.encode(encoding='utf-8')
>>> type(b)
<class 'bytes'>
>>>
>>> b
b'hello world'
>>>
>>>
>>> c = b.decode(encoding='utf-8')
>>> type(c)
<class 'str'>
>>> c
'hello world'

path="doc.txt"
 
f=open(path,'rb')
 
raw=f.read()
print(raw[:100])
print(len(raw))
print(type(raw))
 
raw1=str(raw)
print(raw1[:100])
print(len(raw1))
print(type(raw1))
 
tokens=word_tokenize(raw1)
print(len(tokens))
print(tokens[:20])

b'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash eve'
626168
<class 'bytes'>
b'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash
634857
<class 'str'>
115104
["b'the", 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'century\\', "'s", 'new', '``', 'conan', '``', 'and', 'that', 'he\\', "'s", 'going', 'to']

from nltk import word_tokenize
 
path="doc.txt" 
 
f=open(path,'rb') 
 
raw=f.read() 
print(raw[:1000]) 
print(len(raw)) 
print(type(raw)) 
raw=raw[:1000]
 
 
raw1= raw.decode(encoding='utf-8') #在4000多的位置上有无法解码的，也就是说解码方式是有问题的，应该不是utf-8
print(raw1[:1000]) 
print(len(raw1)) 
print(type(raw1)) 

b'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal . \nthe gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth . \neffective but too-tepid biopic\nif you sometimes like to go to the movies to have fun , wasabi is a good place to start . \nemerges as something rare , an issue movie that\'s so honest and keenly observed that it doesn\'t feel like one . \nthe film provides some great insight into the neurotic mindset of all comics -- even those who have reached the absolute top of the game . \noffers that rare combination of entertainment and education . \nperhaps no picture ever made has more literally showed that the road to hell is paved with good intentions . \nsteers turns in a snappy screenplay that curl'
626168
<class 'bytes'>
the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal . 
the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth . 
effective but too-tepid biopic
if you sometimes like to go to the movies to have fun , wasabi is a good place to start . 
emerges as something rare , an issue movie that's so honest and keenly observed that it doesn't feel like one . 
the film provides some great insight into the neurotic mindset of all comics -- even those who have reached the absolute top of the game . 
offers that rare combination of entertainment and education . 
perhaps no picture ever made has more literally showed that the road to hell is paved with good intentions . 
steers turns in a snappy screenplay that curl
1000
<class 'str'>

189
['e', 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'century', "'s", 'new', '``', 'conan', '``', 'and', 'that', 'he', "'s", 'going', 'to']

>>>a='你好'
>>>b=a.encode('unicode_escape')
>>>b
b'\\u4f60\\u597d'
 
 
>>>a='hello'
>>>b=a.encode('unicode_escape')
>>>b
b'hello'