Evaluate pretrained embeddings#

%config InlineBackend.figure_format='retina'
import warnings
import logging
from ekorpkit import eKonf

logging.basicConfig(level=logging.INFO)
warnings.filterwarnings('ignore')
print(eKonf.__version__)
0.1.32+1.gcf6615f.dirty
data_dir = "../data/embeddings"
save_dataframe = eKonf.partial(
    config_group="_func_/save_dataframe", output_dir=data_dir
)
load_dataframe = eKonf.partial(config_group="_func_/load_dataframe", data_dir=data_dir)

Load GloVE Vectors#

Wikipedia#

cfg = eKonf.compose(config_group="model/embedding")
cfg.name = "glove_wiki"
cfg.corpus = "Wikipedia"
cfg.cache.uri = "https://nlp.stanford.edu/data/glove.6B.zip"
cfg.model_dir = cfg.cache.path
cfg.model_file = "glove.6B.300d.txt"
cfg.model_type = "glove"
wv_wiki = eKonf.instantiate(cfg)
wv_wiki.load()
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.6B.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/d97dcf99e9ed61e3b07a5a87dd5dea7b6a4815ba6eb1ac87f529bfe6e2ba4ccf.f9662ccb99e715467e6da0c85a047cfc51888321958f9440eb5a15f7189c140f-extracted
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.6B.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/d97dcf99e9ed61e3b07a5a87dd5dea7b6a4815ba6eb1ac87f529bfe6e2ba4ccf.f9662ccb99e715467e6da0c85a047cfc51888321958f9440eb5a15f7189c140f-extracted
INFO:cached_path:cache of https://raw.githubusercontent.com/RaRe-Technologies/gensim/develop/gensim/test/test_data/questions-words.txt is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:loading projection weights from /workspace/.cache/cached_path/d97dcf99e9ed61e3b07a5a87dd5dea7b6a4815ba6eb1ac87f529bfe6e2ba4ccf.f9662ccb99e715467e6da0c85a047cfc51888321958f9440eb5a15f7189c140f-extracted/glove.6B.300d.txt
INFO:gensim.utils:KeyedVectors lifecycle event {'msg': 'loaded (400000, 300) matrix of type float32 from /workspace/.cache/cached_path/d97dcf99e9ed61e3b07a5a87dd5dea7b6a4815ba6eb1ac87f529bfe6e2ba4ccf.f9662ccb99e715467e6da0c85a047cfc51888321958f9440eb5a15f7189c140f-extracted/glove.6B.300d.txt', 'binary': False, 'encoding': 'utf8', 'datetime': '2022-06-08T10:14:23.606212', 'gensim': '4.2.0', 'python': '3.8.12 (default, Jan 14 2022, 01:33:56) \n[GCC 7.5.0]', 'platform': 'Linux-5.4.0-58-generic-x86_64-with-glibc2.27', 'event': 'load_word2vec_format'}
wiki_results = wv_wiki.evaluate_word_analogies(analogies="google")
wiki_results.keys()
INFO:gensim.models.keyedvectors:Evaluating word analogies for top 300000 words in the model on /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:capital-common-countries: 94.9% (480/506)
INFO:gensim.models.keyedvectors:capital-world: 96.0% (4342/4524)
INFO:gensim.models.keyedvectors:currency: 17.1% (138/808)
INFO:gensim.models.keyedvectors:city-in-state: 59.3% (1463/2467)
INFO:gensim.models.keyedvectors:family: 88.1% (446/506)
INFO:gensim.models.keyedvectors:gram1-adjective-to-adverb: 22.6% (224/992)
INFO:gensim.models.keyedvectors:gram2-opposite: 27.3% (222/812)
INFO:gensim.models.keyedvectors:gram3-comparative: 88.1% (1174/1332)
INFO:gensim.models.keyedvectors:gram4-superlative: 72.2% (810/1122)
INFO:gensim.models.keyedvectors:gram5-present-participle: 70.0% (739/1056)
INFO:gensim.models.keyedvectors:gram6-nationality-adjective: 92.6% (1480/1599)
INFO:gensim.models.keyedvectors:gram7-past-tense: 61.2% (954/1560)
INFO:gensim.models.keyedvectors:gram8-plural: 78.1% (1040/1332)
INFO:gensim.models.keyedvectors:gram9-plural-verbs: 58.5% (509/870)
INFO:gensim.models.keyedvectors:Quadruplets with out-of-vocabulary words: 0.3%
INFO:gensim.models.keyedvectors:NB: analogies containing OOV words were skipped from evaluation! To change this behavior, use "dummy4unknown=True"
INFO:gensim.models.keyedvectors:Total accuracy: 72.0% (14021/19486)
INFO:ekorpkit.models.embeddings.wordvec:Evaluation score: 0.7195422354510931
dict_keys(['score', 'summary', 'correct', 'incorrect'])

Twitter Data#

cfg = eKonf.compose(config_group="model/embedding")
cfg.name = "glove_twitter"
cfg.corpus = "Twitter"
cfg.cache.uri = "https://nlp.stanford.edu/data/glove.twitter.27B.zip"
cfg.model_dir = cfg.cache.path
cfg.model_file = "glove.twitter.27B.200d.txt"
cfg.model_type = "glove"
wv_twt = eKonf.instantiate(cfg)
wv_twt.load()
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.twitter.27B.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/d08f2221566196fe3fd6f557a2b67e9ffd5cf4b7c500918d98581a52f349a804.7417464dae8ba25e69e639021d320765d2d18bb24f73391ef423756069dc8078-extracted
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.twitter.27B.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/d08f2221566196fe3fd6f557a2b67e9ffd5cf4b7c500918d98581a52f349a804.7417464dae8ba25e69e639021d320765d2d18bb24f73391ef423756069dc8078-extracted
INFO:cached_path:cache of https://raw.githubusercontent.com/RaRe-Technologies/gensim/develop/gensim/test/test_data/questions-words.txt is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:loading projection weights from /workspace/.cache/cached_path/d08f2221566196fe3fd6f557a2b67e9ffd5cf4b7c500918d98581a52f349a804.7417464dae8ba25e69e639021d320765d2d18bb24f73391ef423756069dc8078-extracted/glove.twitter.27B.200d.txt
INFO:gensim.utils:KeyedVectors lifecycle event {'msg': 'loaded (1193514, 200) matrix of type float32 from /workspace/.cache/cached_path/d08f2221566196fe3fd6f557a2b67e9ffd5cf4b7c500918d98581a52f349a804.7417464dae8ba25e69e639021d320765d2d18bb24f73391ef423756069dc8078-extracted/glove.twitter.27B.200d.txt', 'binary': False, 'encoding': 'utf8', 'datetime': '2022-06-08T09:15:08.452346', 'gensim': '4.2.0', 'python': '3.8.12 (default, Jan 14 2022, 01:33:56) \n[GCC 7.5.0]', 'platform': 'Linux-5.4.0-58-generic-x86_64-with-glibc2.27', 'event': 'load_word2vec_format'}
twt_results = wv_twt.evaluate_word_analogies(analogies="google")
twt_results.keys()
INFO:gensim.models.keyedvectors:Evaluating word analogies for top 300000 words in the model on /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:capital-common-countries: 70.6% (357/506)
INFO:gensim.models.keyedvectors:capital-world: 74.6% (1639/2198)
INFO:gensim.models.keyedvectors:currency: 3.3% (15/458)
INFO:gensim.models.keyedvectors:city-in-state: 35.6% (879/2467)
INFO:gensim.models.keyedvectors:family: 79.5% (302/380)
INFO:gensim.models.keyedvectors:gram1-adjective-to-adverb: 12.7% (118/930)
INFO:gensim.models.keyedvectors:gram2-opposite: 30.8% (185/600)
INFO:gensim.models.keyedvectors:gram3-comparative: 74.2% (989/1332)
INFO:gensim.models.keyedvectors:gram4-superlative: 64.0% (718/1122)
INFO:gensim.models.keyedvectors:gram5-present-participle: 66.4% (701/1056)
INFO:gensim.models.keyedvectors:gram6-nationality-adjective: 72.7% (894/1229)
INFO:gensim.models.keyedvectors:gram7-past-tense: 53.1% (787/1482)
INFO:gensim.models.keyedvectors:gram8-plural: 76.0% (1012/1332)
INFO:gensim.models.keyedvectors:gram9-plural-verbs: 52.0% (452/870)
INFO:gensim.models.keyedvectors:Quadruplets with out-of-vocabulary words: 18.3%
INFO:gensim.models.keyedvectors:NB: analogies containing OOV words were skipped from evaluation! To change this behavior, use "dummy4unknown=True"
INFO:gensim.models.keyedvectors:Total accuracy: 56.7% (9048/15962)
INFO:ekorpkit.models.embeddings.wordvec:Evaluation score: 0.5668462598671845
dict_keys(['score', 'summary', 'correct', 'incorrect'])

Common Crawl#

cfg = eKonf.compose(config_group="model/embedding")
cfg.name = "glove_commoncrawl"
cfg.corpus = "Common Crawl"
cfg.cache.uri = "https://nlp.stanford.edu/data/glove.42B.300d.zip"
cfg.model_dir = cfg.cache.path
cfg.model_file = "glove.42B.300d.txt"
cfg.model_type = "glove"
wv_cc = eKonf.instantiate(cfg)
wv_cc.load()
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.42B.300d.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/1b77e307d0976680276bd5eddffac55a83164787611dccb81aaaab9c4c79073b.d4304a004cdd3b0267ad688b5fc4d6c9d43c622a571ee2ba8b39604fc257faed-extracted
INFO:cached_path:cache of https://nlp.stanford.edu/data/glove.42B.300d.zip is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /workspace/.cache/cached_path/1b77e307d0976680276bd5eddffac55a83164787611dccb81aaaab9c4c79073b.d4304a004cdd3b0267ad688b5fc4d6c9d43c622a571ee2ba8b39604fc257faed-extracted
INFO:cached_path:cache of https://raw.githubusercontent.com/RaRe-Technologies/gensim/develop/gensim/test/test_data/questions-words.txt is up-to-date
INFO:ekorpkit.io.cached_path:cached path: /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:loading projection weights from /workspace/.cache/cached_path/1b77e307d0976680276bd5eddffac55a83164787611dccb81aaaab9c4c79073b.d4304a004cdd3b0267ad688b5fc4d6c9d43c622a571ee2ba8b39604fc257faed-extracted/glove.42B.300d.txt
INFO:gensim.utils:KeyedVectors lifecycle event {'msg': 'loaded (1917494, 300) matrix of type float32 from /workspace/.cache/cached_path/1b77e307d0976680276bd5eddffac55a83164787611dccb81aaaab9c4c79073b.d4304a004cdd3b0267ad688b5fc4d6c9d43c622a571ee2ba8b39604fc257faed-extracted/glove.42B.300d.txt', 'binary': False, 'encoding': 'utf8', 'datetime': '2022-06-08T09:22:21.176032', 'gensim': '4.2.0', 'python': '3.8.12 (default, Jan 14 2022, 01:33:56) \n[GCC 7.5.0]', 'platform': 'Linux-5.4.0-58-generic-x86_64-with-glibc2.27', 'event': 'load_word2vec_format'}
cc_results = wv_cc.evaluate_word_analogies(analogies="google")
cc_results.keys()
INFO:gensim.models.keyedvectors:Evaluating word analogies for top 300000 words in the model on /root/.ekorpkit/.cache/cached_path/7568e1428a2ba53a33f67dea7e276f212c065b07d102aafc177a507d244161f6.f60f7f82aaacf3a534d93013617bcfead46a795268096ff1990608b16115b566
INFO:gensim.models.keyedvectors:capital-common-countries: 95.1% (481/506)
INFO:gensim.models.keyedvectors:capital-world: 94.0% (4178/4446)
INFO:gensim.models.keyedvectors:currency: 17.6% (142/808)
INFO:gensim.models.keyedvectors:city-in-state: 78.1% (1926/2467)
INFO:gensim.models.keyedvectors:family: 90.9% (460/506)
INFO:gensim.models.keyedvectors:gram1-adjective-to-adverb: 30.2% (300/992)
INFO:gensim.models.keyedvectors:gram2-opposite: 35.6% (289/812)
INFO:gensim.models.keyedvectors:gram3-comparative: 85.6% (1140/1332)
INFO:gensim.models.keyedvectors:gram4-superlative: 84.0% (942/1122)
INFO:gensim.models.keyedvectors:gram5-present-participle: 80.9% (854/1056)
INFO:gensim.models.keyedvectors:gram6-nationality-adjective: 88.3% (1412/1599)
INFO:gensim.models.keyedvectors:gram7-past-tense: 49.3% (769/1560)
INFO:gensim.models.keyedvectors:gram8-plural: 84.9% (1131/1332)
INFO:gensim.models.keyedvectors:gram9-plural-verbs: 64.1% (558/870)
INFO:gensim.models.keyedvectors:Quadruplets with out-of-vocabulary words: 0.7%
INFO:gensim.models.keyedvectors:NB: analogies containing OOV words were skipped from evaluation! To change this behavior, use "dummy4unknown=True"
INFO:gensim.models.keyedvectors:Total accuracy: 75.1% (14582/19408)
INFO:ekorpkit.models.embeddings.wordvec:Evaluation score: 0.7513396537510305
dict_keys(['score', 'summary', 'correct', 'incorrect'])

Evaluate embeddings#

import pandas as pd

eval_summary = pd.concat(
    [wiki_results["summary"], twt_results["summary"], cc_results["summary"]]
)
save_dataframe(eval_summary, output_file="eval_summary.parquet")
INFO:ekorpkit.io.file:Saving dataframe as ../data/embeddings/eval_summary.parquet
eval_summary = load_dataframe(data_file="eval_summary.parquet")
eval_summary.head()
INFO:ekorpkit.io.file:Loading data from ../data/embeddings/eval_summary.parquet
Category Correct Incorrect Samples Average Corpus
0 Capitals 480 26 506 0.948617 Wikipedia
1 Capitals RoW 4342 182 4524 0.959770 Wikipedia
2 Currency 138 670 808 0.170792 Wikipedia
3 City-State 1463 1004 2467 0.593028 Wikipedia
4 Famliy 446 60 506 0.881423 Wikipedia
cfg = eKonf.compose(config_group="visualize/plot=barplot")
cfg.plots[0].x = "Category"
cfg.plots[0].y = "Average"
cfg.plots[0].hue = "Corpus"
cfg.ax.ytickmajorformatterfunc = "lambda y, _: '{:.0%}'.format(y)"
cfg.ax.ylabel = "Accuracy"
cfg.figure.figsize = (16, 5)
cfg.figure.fontsize = 10
cfg.ax.title = f"Word Vector Accuracy by Glove Source: Twitter: {twt_results['score']:.2%}, Wiki: {wiki_results['score']:.2%}, Crawl: {cc_results['score']:.2%}"
eKonf.instantiate(cfg, data=eval_summary)
INFO:ekorpkit.visualize.plot:Plotting barplot with {'x': 'Category', 'y': 'Average', 'hue': 'Corpus'}
INFO:ekorpkit.visualize.plot:Saved figure to ./figs/plot_BarPlot.png
../../../../_images/d18ca3d98c465059f54f7573de8207b2384a384eaa9ad39af778225036eb12e1.png

Visualize Embeddings#

results = wv_wiki.reduce_embeddings_2d(restrict_vocab=100_000)
vectors = results["vectors"]
word2idx = results["word2idx"]
INFO:ekorpkit.models.embeddings.wordvec:dimensions: (100000, 300)
INFO:ekorpkit.models.embeddings.wordvec:explained variance: [0.02604632 0.01293811]
best_analogies = wv_wiki.find_most_similar_analogies(
    wiki_results["correct"], word2idx, vectors
)
best_analogies
wordpairs word2idx similarity
category
Adj-Adverb (fortunate, fortunately, lucky, luckily) (11156, 11584, 5065, 19955) 1.998511
Capitals (london, england, paris, france) (516, 563, 1035, 387) 1.780721
Capitals RoW (vienna, austria, brussels, belgium) (4094, 2640, 3879, 2975) 1.996072
City-State (chicago, illinois, omaha, nebraska) (1147, 2884, 12159, 6087) 1.971419
Comparative (long, longer, heavy, heavier) (173, 1078, 1106, 11613) 1.932268
Currency (usa, dollar, russia, ruble) (2396, 678, 412, 17506) 1.834880
Famliy (sons, daughters, stepfather, stepmother) (2912, 4321, 20624, 26903) 1.999805
Nationality (switzerland, swiss, israel, israeli) (2311, 1849, 315, 406) 1.999926
Opposite (tasteful, distasteful, likely, unlikely) (43255, 41259, 647, 2993) 1.954353
Past Tense (sitting, sat, hitting, hit) (2995, 3223, 3141, 416) 1.999999
Plural (eye, eyes, donkey, donkeys) (2090, 2251, 20328, 35193) 1.999876
Plural Verbs (talk, talks, estimate, estimates) (1077, 370, 3470, 2886) 1.739114
Pres. Part. (write, writing, read, reading) (2432, 1649, 1465, 2185) 1.999959
Superlative (weak, weakest, big, biggest) (2690, 15655, 365, 882) 0.663122
Total accuracy (sitting, sat, hitting, hit) (2995, 3223, 3141, 416) 1.999999

Plot Analogy Examples#

wv_wiki.plot_similar_analogies(best_analogies, vectors, ncols=3, figsize=(15, 15))
INFO:ekorpkit.visualize.plot:No data to plot
INFO:ekorpkit.visualize.plot:No plots to plot
INFO:ekorpkit.visualize.plot:Saved figure to ./figs/plot_plot.png
../../../../_images/78fd5bd8e6a2299238d54bc41ee4d279c42f4823fd8058d688a59a8fea9452f7.png