intial state

Author: mikeraf

classify.py

@@ -0,0 +1,22 @@
+from gensim.models import word2vec
+from sklearn.cluster import KMeans
+
+#cd PycharmProjects/play_gensim/
+
+w2v = word2vec.Word2Vec.load(u'C:\\Users\\michar\\PycharmProjects\\play_gensim\\w2v_model_size_100_window_5.model')
+
+kmeans = KMeans(n_clusters=50, random_state=0).fit(w2v.syn0)
+
+
+labels_count = {}
+for l in kmeans.labels_:
+    labels_count[l] = labels_count.get(l,0)+1
+for l in kmeans.labels_:
+    labels_count[l] = labels_count.get(l,0)+1
+
+sorted_labels = sorted(labels_count.keys(), key = labels_count.get)
+
+minimal_label = sorted_labels[2]
+for i,l in enumerate(kmeans.labels_):
+    if l == minimal_label:
+        print w2v.index2word[i]

main.py

@@ -0,0 +1,29 @@
+
+from gensim.models import word2vec
+import time
+
+
+def convert_corpora_to_sentence_iter():
+    t8 = word2vec.Text8Corpus('text8/text8')
+    return t8
+
+
+def train_and_save_model(sentences, fname, **kwargs):
+    w2v = word2vec.Word2Vec(sentences, **kwargs)
+    w2v.save(fname)
+    return w2v
+
+def get_model_name(**kw):
+    return u"w2v_model_size_{model_size}_window_{window}.model".format(**kw)
+
+print __name__
+if __name__ == '__main__':
+    sentences_iter = convert_corpora_to_sentence_iter()
+    for model_size in range(10, 200, 10):
+        for window in (5, 7, 10):
+            t0 = time.clock()
+            print "Training with size={}, window={}".format(model_size, window)
+            fname = get_model_name(model_size=model_size, window=window)
+            model = train_and_save_model(sentences_iter, fname, size=model_size, window=window)
+            t1 = time.clock()
+            print "Training took {} secs".format(t1-t0)

meassure_convergence.py

@@ -0,0 +1,20 @@
+from main import get_model_name
+from gensim.models import word2vec
+from sklearn.cluster import KMeans
+from sklearn.metrics import homogeneity_completeness_v_measure as hom_v_score
+
+NCLUSTERS = 50
+results = {}
+for window_size in (5,7,10):
+    results[window_size] = []
+    ref_model_name = get_model_name(model_size=190, window=window_size)
+    ref_model = word2vec.Word2Vec.load(u'C:\\Users\\michar\\PycharmProjects\\play_gensim\\'+ref_model_name)
+    ref_kmeans = KMeans(n_clusters=NCLUSTERS, random_state=0).fit(ref_model.syn0)
+    for model_size in range(10, 200, 10):
+        model_name = get_model_name(model_size=model_size, window=window_size)
+        model = word2vec.Word2Vec.load(u'C:\\Users\\michar\\PycharmProjects\\play_gensim\\'+model_name)
+        kmeans =  KMeans(n_clusters=NCLUSTERS, random_state=0).fit(model.syn0)
+
+        score = hom_v_score(kmeans.labels_, ref_kmeans.labels_)[2]
+        print "window {}, size {}, score {}".format(window_size, model_size, score)
+        results[window_size].append((model_size, score))