하위 단어(Sub-Word)정보를 통한 단어 벡터(Word Vectors) 강화

>>>word = "where"
>>>word = f"<{word}>"
>>>n_grams = [word[i:i+3] for i in range(len(word)-2)]+[word]
n_grams
>>>n_grams
['<wh', 'whe', 'her', 'ere', 're>', '<where>']

# Create a `tf.data` with all the non-negative sentences
>>> text_ds = tf.data.TextLineDataset(path_to_file).filter(lambda x: tf.cast(tf.strings.length(x), bool))

>>> for text in text_ds.take(5):
    	print(text)

tf.Tensor(b'First Citizen:', shape=(), dtype=string)
tf.Tensor(b'Before we proceed any further, hear me speak.', shape=(), dtype=string)
tf.Tensor(b'All:', shape=(), dtype=string)
tf.Tensor(b'Speak, speak.', shape=(), dtype=string)
tf.Tensor(b'First Citizen:', shape=(), dtype=string) 

# We create a custom standardization function to lowercase the text and 
# remove punctuation.
def custom_standardization(input_data):
    lowercase = tf.strings.lower(input_data)
    return tf.strings.regex_replace(lowercase,
                                    '[%s]' % re.escape(string.punctuation), '')

# Define the vocabulary size and number of words in a sequence.
vocab_size = 4096
sequence_length = 10

# Use the text vectorization layer to normalize, split, and map strings to
# integers. Set output_sequence_length length to pad all samples to same length.
vectorize_layer = TextVectorization(
    standardize=custom_standardization,
    max_tokens=vocab_size,
    output_mode='int',
    output_sequence_length=sequence_length)

# build the vocab
vectorize_layer.adapt(text_ds.batch(1024))

class Word2Vec(Model):
    def __init__(self, subword_vocab_size, vocab_size, embedding_dim):
        super(Word2Vec, self).__init__()
        self.target_embedding = Embedding(subword_vocab_size+1, 
                                        embedding_dim,
                                        input_length=None,
                                        name="w2v_embedding",)
        self.context_embedding = Embedding(vocab_size+1, 
                                        embedding_dim, 
                                        input_length=num_ns+1)
        self.dots = Dot(axes=(3,1))
        self.flatten = Flatten()

    def call(self, pair):
        target, context = pair
        we = tf.math.reduce_sum(self.target_embedding(target),axis=1)
        ce = self.context_embedding(context)
        dots = self.dots([ce, we])
        return self.flatten(dots)