@@ -14,119 +14,4 @@ limitations under the License. */
1414
1515#pragma once
1616
17- #include < codecvt>
18- #include < iostream>
19- #include < locale>
20- #include < string>
21- #include < unordered_map>
22- #include < vector>
23- #include " paddle/fluid/framework/phi_tensor_base_vector.h"
24- #include " paddle/phi/core/dense_tensor.h"
25- #include " paddle/phi/core/extended_tensor.h"
26-
27- namespace paddle {
28- namespace framework {
29-
30- // Note(YuanRisheng): Vocab is mainly used for faster_tokenizer_op and we don't
31- // recommend widely use it. Because faster_tokenizer_op may be deleted in the
32- // future and this class will be deleted.
33-
34- class Vocab : public phi ::ExtendedTensor,
35- public phi::TypeInfoTraits<phi::TensorBase, Vocab> {
36- public:
37- Vocab () = default ;
38-
39- Vocab (Vocab&& other) = default ;
40-
41- Vocab (const Vocab& other) = default ;
42-
43- Vocab& operator =(const Vocab& other) = default ;
44-
45- Vocab& operator =(Vocab&& other) = default ;
46-
47- Vocab& operator =(
48- const std::unordered_map<std::wstring, std::int32_t >& other) {
49- this ->data_ = other;
50- return *this ;
51- }
52-
53- // / \brief Destroy the Vocab and release exclusive resources.
54- virtual ~Vocab () = default ;
55-
56- public:
57- // / \brief Returns the name of the class for type traits.
58- // / \return The name of the class.
59- static const char * name () { return " Vocab"
60-
61- size_t size () const { return data_.size (); }
62-
63- void clear () { data_.clear (); }
64-
65- void emplace (const std::wstring& key, std::int32_t value) {
66- data_.emplace (key, value);
67- }
68-
69- std::int32_t at (const std::wstring& key) { return data_.at (key); }
70-
71- std::int32_t at (const std::wstring& key) const { return data_.at (key); }
72-
73- std::unordered_map<std::wstring, std::int32_t >::iterator find (
74- const std::wstring& key) {
75- return data_.find (key);
76- }
77-
78- std::unordered_map<std::wstring, std::int32_t >::const_iterator find (
79- const std::wstring& key) const {
80- return data_.find (key);
81- }
82-
83- std::unordered_map<std::wstring, std::int32_t >::iterator begin () {
84- return data_.begin ();
85- }
86-
87- std::unordered_map<std::wstring, std::int32_t >::const_iterator begin () const {
88- return data_.begin ();
89- }
90-
91- std::unordered_map<std::wstring, std::int32_t >::iterator end () {
92- return data_.end ();
93- }
94-
95- std::unordered_map<std::wstring, std::int32_t >::const_iterator end () const {
96- return data_.end ();
97- }
98-
99- private:
100- std::unordered_map<std::wstring, std::int32_t > data_;
101- };
102-
103- // Note(YuanRisheng): PhiVector is essentially a vector that only used for PHI
104- // Kernel. It can be used when you define a non-tensor type that needs to be
105- // stored in a vector as PHI kernel argument.
106-
107- template <>
108- struct PhiVectorType <std::string> {
109- const char * type_name = " PhiVectorString"
110- };
111-
112- using String = std::string;
113- using Strings = PhiVector<std::string>;
114-
115- // Convert the std::string type to the std::string type.
116- bool ConvertStrToWstr (const std::string& src, std::wstring* res);
117- // Convert the std::wstring type to the std::string type.
118- void ConvertWstrToStr (const std::wstring& src, std::string* res);
119- // Normalization Form Canonical Decomposition.
120- void NFD (const std::string& s, std::string* ret);
121-
122- // Write the data which is type of
123- // std::unordered_map<td::string, int32_t> to ostream.
124- void StringMapToStream (std::ostream& os,
125- const std::unordered_map<std::string, int32_t >& data);
126-
127- // Read the data which is type of
128- // std::unordered_map<td::string, int32_t> from istream.
129- void StringMapFromStream (std::istream& is,
130- std::unordered_map<std::string, int32_t >* data);
131- } // namespace framework
132- } // namespace paddle
17+ #include "paddle/phi/core/vocab/string_array.h"
0 commit comments