e2e

Fill in a module description here

Embedding Projector

import io
import json
import math
import os

import numpy as np
import pandas as pd
import tensorflow as tf
from tensorboard.plugins import projector

from portfolio_search.fileorg import *
singleline_data_home()
Path('/Users/al/singleline_data')

Loading TSV’s and Centroids

raster_epoch_dir = singleline_data_home() / "raster/epoch-20231214/"
subclusters_dir = (
    raster_epoch_dir / "04_SUBCLUSTERS"
)
drawing_centroids_fname = f"{subclusters_dir}/centroids_drawings.json"
with open(drawing_centroids_fname, "r") as infile:
    drawing_centroids = json.load(infile)
watercolor_centroids_fname = f"{subclusters_dir}/centroids_watercolors.json"
with open(watercolor_centroids_fname, "r") as infile:
    watercolor_centroids = json.load(infile)
clustered_drawings_tsv_fname = f"{subclusters_dir}/clustered_drawings.tsv"
# drawings_df.to_csv(
#     clustered_drawings_tsv_fname, index=True, index_label="idx", sep="\t", header=True
# )
clustered_drawings_df = pd.read_csv(
    clustered_drawings_tsv_fname, delimiter="\t", index_col="idx"
)
clustered_drawings_df.head()
abs_fname rel_fname label pred_label pred_idx pred_probs emb_csv cluster cluster_dist metacluster ... drawings_cluster48_id drawings_cluster48_dist drawings_cropped_cluster16_id drawings_cropped_cluster16_dist drawings_cropped_cluster32_id drawings_cropped_cluster32_dist drawings_cropped_cluster48_id drawings_cropped_cluster48_dist drawings_cropped_cluster64_id drawings_cropped_cluster64_dist
idx
0 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb77p043.jpg art/sb77p043.jpg art xtra 4 0.168532,0.008473,0.008068,0.016940,0.797986 -0.6198182,-0.5714155,-0.6871516,-0.6553513,-0.6254566,-0.65341413,-0.7267234,-0.4896673,-0.7537334,-0.42715576,1.4904574,-0.72562873,-0.662768,-0.6266761,0.6913696,0.054546177,-0.5467274,-0.6520286,0.117179275,0.94330966,0.5815062,-0.55193967,-0.5446661,-0.6779002,-0.57759243,-0.4584825,-0.3818994,-0.5750927,-0.73986673,-0.45211998,-0.6141643,-0.67007643,-0.52171326,-0.4592305,1.4788612,-0.6186755,0.799198,-0.57338566,-0.17778617,0.10336053,0.093812525,0.17924881,-0.6189488,-0.6682474,2.1040225,-0.59257406,1.6553459,-0.6001429,0.5109105,0.19352531,-0.2517513,-0.66552746,-0.6482407,-0.4732... 11 199.22961 1 ... 15 166.33705 15 265.844100 15 257.350560 15 268.67297 58 276.82034
1 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb67p021.jpg art/sb67p021.jpg art xtra 4 0.170976,0.008673,0.008480,0.017028,0.794843 1.1073987,0.3512016,-0.73929733,-0.6553513,1.7477593,-0.65341413,0.32179314,-0.4896673,-0.1797334,-0.42715576,1.3163843,-0.72562873,-0.61387855,-0.6266761,-0.35361218,0.37611246,-0.08771938,-0.5262803,0.9261699,-0.6497359,-0.64108604,-0.55193967,-0.5446661,-0.6779002,-0.32559583,-0.6690378,-0.4518578,0.51828945,-0.4238917,-0.45211998,-0.16527033,-0.67007643,1.0136865,-0.4592305,-0.600525,-0.060860336,-0.19446748,-0.24127388,2.2282472,1.8003348,-0.61049384,0.39795423,0.2626753,-0.6682474,-0.75794554,-0.59257406,0.80257696,-0.6001429,-0.651804,-0.028328001,0.53641737,-0.6501347,-0.6482407,0.... 3 132.26111 0 ... 45 137.20117 14 121.146545 31 122.172424 31 116.40921 34 122.04242
2 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb25p227.jpg art/sb25p227.jpg art xtra 4 0.158396,0.008277,0.007424,0.016584,0.809319 1.3776337,-0.5714155,-0.2587232,-0.6553513,1.3532461,-0.65341413,-0.7267234,-0.4896673,-0.7537334,-0.25077122,0.050992966,-0.72562873,0.1406585,-0.6266761,1.0653365,0.96198404,-0.5467274,0.30261123,0.6327609,-0.6497359,-0.64108604,-0.28434965,-0.35651857,-0.6779002,-0.28245115,-0.6690378,-0.4518578,0.030088186,-0.73986673,-0.18931103,-0.041653574,-0.67007643,-0.52171326,-0.29881266,-0.600525,-0.23231092,-0.73933,-0.57338566,-0.65925777,1.8833572,-0.61049384,0.44560486,0.14075392,-0.6682474,-0.75794554,-0.59257406,-0.07511848,-0.6001429,-0.651804,-0.5315948,0.023931146,0.723624,-0.6482407,-... 8 188.58075 0 ... 8 148.63470 10 220.298950 1 217.335360 1 213.25168 1 191.67163
3 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb48p057.jpg art/sb48p057.jpg art xtra 4 0.165322,0.008453,0.008010,0.016587,0.801628 2.4936543,0.17563903,-0.6522293,-0.55642915,1.1084181,-0.5028205,-0.046447277,-0.4896673,-0.7537334,-0.42715576,0.84124434,-0.72562873,-0.52445275,-0.5557985,-0.21369287,2.2372599,-0.26381853,-0.6520286,-0.6898426,-0.6497359,-0.64108604,0.43080842,-0.23286673,-0.6779002,0.7969176,-0.6690378,-0.4518578,0.747658,0.04896885,-0.45211998,-0.68509877,-0.2462106,0.7319356,-0.4592305,-0.600525,-0.6186755,-0.5338096,-0.57338566,1.1886733,0.04046011,-0.61049384,0.45326668,0.72983754,-0.6682474,-0.10000485,0.034278154,-0.33160114,-0.6001429,-0.651804,-0.5315948,-0.20898154,-0.66552746,1.2346795,-0.47... 14 90.26291 0 ... 28 83.96362 2 144.254030 21 129.951970 21 140.07254 21 144.77570
4 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb77p044.jpg art/sb77p044.jpg art xtra 4 0.171638,0.008182,0.008061,0.016469,0.795650 2.170152,-0.01916939,-0.47766963,-0.6553513,-0.44795185,-0.65341413,1.5596724,-0.4896673,-0.7537334,-0.42715576,1.0671424,-0.72562873,1.8207353,-0.6266761,-0.6194963,1.1724913,-0.5467274,0.12343103,1.5514618,-0.6497359,0.23890108,-0.55193967,-0.5446661,-0.6779002,-0.57759243,0.33040416,-0.4518578,0.39869583,0.120936334,-0.45211998,-0.68509877,0.2514736,0.17836922,-0.4592305,-0.600525,-0.6186755,-0.73933,-0.57338566,1.3668395,0.6006557,1.2428443,1.0844817,0.36636358,0.48543411,-0.5954572,-0.59257406,-0.30289415,1.2783914,0.5800507,-0.23473695,-0.59497476,-0.66552746,2.2758756,-0.4732165,-0.... 4 176.29102 0 ... 39 169.47894 15 151.277310 15 141.268980 15 145.39337 58 151.13809

5 rows × 41 columns

clustered_watercolors_tsv_fname = f"{subclusters_dir}/clustered_watercolors.tsv"
clustered_watercolors_df = pd.read_csv(
    clustered_watercolors_tsv_fname, delimiter="\t", index_col="idx"
)
clustered_watercolors_df.head()
abs_fname rel_fname label pred_label pred_idx pred_probs emb_csv cluster cluster_dist metacluster ... handlabeled_metacluster_name handlabeled_metacluster_id handlabeled_metacluster_was_correct preprocessed_abs_path handlabeled_abs_path emb_csv_cropped watercolors_cluster_id watercolors_cluster_dist watercolors_cropped_cluster_id watercolors_cropped_cluster_dist
idx
0 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb07p104.jpg art/sb07p104.jpg art xtra 4 0.177406,0.008799,0.008245,0.017177,0.788373 -0.39105213,0.96309596,-0.73929733,0.4489988,0.8297614,-0.3937308,-0.7267234,-0.32461396,-0.7537334,-0.42715576,-0.62038505,-0.72562873,0.3037846,-0.6266761,-0.5213815,-0.31669965,-0.5467274,-0.6520286,-0.22544897,-0.6497359,-0.64108604,0.6374447,-0.5446661,-0.6779002,-0.14143991,-0.6690378,-0.44753593,0.66945183,-0.73986673,-0.45211998,-0.05996394,-0.21525535,0.82928157,-0.04412651,-0.600525,-0.04148197,-0.73825717,-0.57338566,-0.25227422,2.3566113,-0.61049384,1.0027854,-0.40134108,1.3733256,-0.75794554,-0.59257406,-0.4395525,-0.6001429,1.0358506,1.1911299,0.8162154,0.032713354,-0.6482407... 5 238.33170 1 ... 1_watercolors 1 True /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/10_SIMPLE_CROP/1_watercolors/sb07p104.jpg /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/03_VISUAL_HAND_LABELED/1_watercolors/sb07p104.jpg -0.6198182,0.8895926,-0.73929733,0.6418346,0.46164584,1.0085131,-0.7267234,-0.4896673,-0.7537334,-0.42715576,-0.62038505,-0.72562873,1.4399182,-0.6266761,0.474914,-0.57515705,-0.5467274,-0.6520286,1.2274435,-0.6497359,-0.27622125,-0.45870692,-0.5446661,-0.6779002,1.3343718,-0.6690378,-0.4518578,0.42277813,-0.73986673,-0.45211998,1.6291834,-0.67007643,-0.09817025,-0.00090405345,-0.600525,-0.6186755,-0.73933,-0.57338566,0.006386578,1.034553,-0.61049384,-0.58021957,-0.09483403,0.78779536,-0.73080033,0.10947651,0.3367887,-0.5382227,-0.26548737,0.5630663,2.15088,0.2266624,-0.6482407,0.049348265... 7 215.02927 7 206.64117
1 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb31p017.jpg art/sb31p017.jpg art xtra 4 0.172119,0.008924,0.008226,0.016950,0.793781 -0.05104202,-0.3935787,-0.73929733,0.86878717,-0.6254566,-0.65341413,-0.7267234,-0.4896673,-0.7537334,-0.14799604,-0.62038505,-0.72562873,-0.2654881,-0.6266761,-0.68991846,-0.57515705,-0.41532665,-0.6520286,-0.6898426,-0.34099144,-0.64108604,-0.55193967,-0.5446661,-0.6779002,-0.2674706,-0.53598946,-0.13334358,-0.5982413,-0.73986673,-0.45211998,-0.68509877,0.033780396,-0.52171326,0.054827005,-0.20721096,0.3301326,-0.73933,-0.57338566,0.34032136,-0.6179389,-0.20170414,0.16225064,-0.44437563,0.44581896,-0.069196045,0.20976627,-0.07846111,-0.6001429,0.38122594,-0.09738222,-0.59497476,-0.665527... 11 191.66959 1 ... 1_watercolors 1 True /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/10_SIMPLE_CROP/1_watercolors/sb31p017.jpg /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/03_VISUAL_HAND_LABELED/1_watercolors/sb31p017.jpg 1.491735,-0.5714155,-0.73929733,0.05867392,0.5154146,0.0936929,-0.7267234,-0.4896673,0.4246894,-0.42715576,-0.33477074,-0.72562873,1.2274611,-0.6266761,-0.68991846,-0.57515705,-0.5467274,1.1312869,-0.06054777,-0.028933167,-0.64108604,-0.55193967,-0.5446661,-0.6779002,-0.57759243,0.46393585,0.4817328,0.22738433,-0.73986673,-0.45211998,0.0057709217,-0.67007643,-0.52171326,0.68283725,0.060967267,0.7253895,-0.73933,-0.57338566,-0.65925777,-0.6179389,-0.32987997,0.86336035,-0.6189488,1.3613329,1.2250904,-0.59257406,-0.5857397,-0.6001429,-0.5687719,-0.14814761,0.5325973,-0.66552746,-0.6482407,0.... 4 199.03098 4 195.97546
2 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb05p079.jpg art/sb05p079.jpg art xtra 4 0.172158,0.008873,0.008536,0.017017,0.793416 1.8089037,-0.5714155,-0.73929733,-0.10773122,-0.22753507,0.34884918,0.124596715,-0.4896673,-0.27447522,-0.42715576,0.61659455,-0.03226477,-0.662768,-0.6266761,-0.68991846,-0.57515705,-0.39897114,-0.6520286,-0.6898426,-0.6497359,0.16184372,-0.55193967,-0.5446661,-0.6779002,0.04647112,-0.6690378,-0.4518578,0.9576235,0.29423904,-0.45211998,-0.68509877,0.09260088,-0.3948297,-0.4592305,-0.600525,-0.6186755,-0.64046603,-0.57338566,0.08902073,0.09568572,-0.61049384,0.7515181,-0.15892029,-0.4098586,1.8483992,-0.59257406,-0.6157326,-0.56642026,-0.651804,-0.44185895,-0.19970179,0.19587982,1.3608505,... 11 140.32997 1 ... 1_watercolors 1 NaN /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/10_SIMPLE_CROP/1_watercolors/sb05p079.jpg /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/03_VISUAL_HAND_LABELED/1_watercolors/sb05p079.jpg -0.6013173,-0.5714155,-0.7368871,-0.6553513,-0.6254566,-0.65341413,0.071327925,-0.4896673,-0.7537334,-0.29406685,1.2854458,-0.72562873,1.4279991,-0.2084853,0.4439276,-0.57515705,-0.5467274,-0.2426523,0.6567551,-0.6497359,0.284827,-0.55193967,-0.5446661,-0.6779002,-0.57759243,-0.6690378,0.39760548,-0.5982413,-0.16403872,0.82081807,-0.68509877,-0.67007643,-0.16007048,-0.23898152,0.4340614,-0.6186755,-0.73933,-0.57338566,-0.65925777,-0.60700464,-0.22504222,1.2887287,0.17025876,-0.5827259,3.245521,-0.59257406,1.607188,-0.6001429,-0.1800242,-0.5315948,0.2059763,0.33526373,0.109642506,-0.0013180... 4 142.76602 4 248.37231
3 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb26p082.jpg art/sb26p082.jpg art xtra 4 0.179525,0.008828,0.008282,0.017196,0.786169 1.8392665,0.3796271,-0.73929733,0.82189727,1.6830277,0.9302285,-0.5532687,-0.4896673,-0.30764756,-0.42715576,-0.44450876,-0.72562873,-0.662768,0.54702616,-0.68991846,-0.57515705,-0.31611928,-0.59368926,1.2819774,-0.6497359,-0.64108604,-0.55193967,0.84984857,-0.6779002,-0.06536412,-0.6690378,0.34403068,-0.16627067,0.3990196,-0.45211998,0.65170527,-0.67007643,0.6049255,-0.4367706,-0.600525,-0.0046679974,-0.73933,-0.109048486,-0.65925777,0.65358853,-0.61049384,0.70374626,-0.6189488,-0.6682474,-0.0177418,-0.59257406,1.1068444,-0.29651338,-0.651804,-0.5315948,1.8200746,-0.66552746,0.11623299,-0... 7 176.11029 1 ... 1_watercolors 1 NaN /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/10_SIMPLE_CROP/1_watercolors/sb26p082.jpg /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/03_VISUAL_HAND_LABELED/1_watercolors/sb26p082.jpg 0.8175719,0.48385304,-0.73929733,1.0291914,1.4682066,1.225123,-0.43518582,-0.4896673,-0.7537334,-0.42715576,0.72162604,-0.72562873,-0.662768,-0.6266761,-0.55470747,-0.57515705,-0.5467274,-0.6520286,2.099903,-0.6497359,-0.64108604,-0.55193967,1.3572993,-0.6779002,1.3149762,0.8512635,0.81857395,-0.5982413,0.056738913,-0.45211998,2.0946088,-0.67007643,-0.52171326,-0.4592305,-0.600525,1.1100725,-0.73933,-0.57338566,-0.24140868,0.9004108,-0.61049384,0.4742828,-0.6189488,-0.6682474,0.335356,0.40166396,0.7790335,-0.6001429,-0.651804,-0.4778452,0.7360641,-0.66552746,-0.6482407,-0.13845,-0.7311095,... 1 157.93832 1 196.84943
4 /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/01_FLAT/art/sb69p101-color.jpg art/sb69p101-color.jpg art xtra 4 0.179525,0.008828,0.008282,0.017196,0.786169 -0.23259398,-0.5714155,-0.73929733,-0.6553513,0.34417403,1.046661,-0.42288247,-0.4896673,0.39017296,-0.42715576,-0.62038505,-0.72562873,-0.662768,-0.6266761,0.93892807,-0.57515705,-0.5222921,-0.6520286,-0.6898426,-0.6497359,-0.6366938,-0.55193967,-0.5446661,-0.6779002,-0.46687827,-0.6690378,-0.4518578,-0.5982413,0.7252437,-0.45211998,1.8137504,-0.67007643,-0.52171326,-0.36457688,-0.600525,0.8178272,-0.73933,0.23574609,-0.65925777,-0.6179389,-0.61049384,-0.15158218,-0.41958112,-0.6682474,-0.75794554,0.4943592,0.8210626,-0.6001429,-0.651804,-0.080083996,0.48374963,-0.42743486,-0.6482407,-0.4... 7 187.45346 1 ... 1_watercolors 1 NaN /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/10_SIMPLE_CROP/1_watercolors/sb69p101-color.jpg /Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/03_VISUAL_HAND_LABELED/1_watercolors/sb69p101-color.jpg 0.14908737,-0.5714155,-0.73929733,0.44245207,-0.6254566,1.4611686,-0.7267234,-0.4896673,0.87923527,-0.42715576,-0.62038505,-0.72562873,-0.662768,-0.6266761,0.4918049,-0.57515705,-0.5467274,-0.6520286,-0.6898426,-0.2860955,-0.64108604,-0.55193967,-0.5446661,-0.6779002,0.69009775,-0.6690378,-0.4518578,0.18292433,-0.004632473,-0.45211998,1.7543458,-0.67007643,-0.52171326,-0.41669255,-0.600525,1.4654884,0.5742926,-0.57338566,-0.65925777,-0.6179389,-0.52983093,0.26484245,-0.25630984,-0.6682474,0.19898295,0.95297796,0.38033813,-0.6001429,-0.651804,-0.5315948,1.3264011,0.31547034,-0.6482407,-0.47... 1 167.27823 1 136.63800

5 rows × 25 columns

# %load_ext tensorboard
# import csv

# import numpy as np
# import tensorflow as tf
# from PIL import Image
drawings_emb = np.stack(
    [
        np.array([float(f) for f in s.split(",")])
        for s in list(clustered_drawings_df.emb_csv)
    ]
).astype(np.float32)

drawings_cropped_emb = np.stack(
    [
        np.array([float(f) for f in s.split(",")])
        for s in list(clustered_drawings_df.emb_csv_cropped)
    ]
).astype(np.float32)

drawings_emb.shape, drawings_cropped_emb.shape
((1923, 512), (1923, 512))
watercolors_emb = np.stack(
    [
        np.array([float(f) for f in s.split(",")])
        for s in list(clustered_watercolors_df.emb_csv)
    ]
).astype(np.float32)

watercolors_cropped_emb = np.stack(
    [
        np.array([float(f) for f in s.split(",")])
        for s in list(clustered_watercolors_df.emb_csv_cropped)
    ]
).astype(np.float32)

watercolors_emb.shape, watercolors_cropped_emb.shape
((404, 512), (404, 512))
def _shim(s):
    return s.replace('/Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2', str(raster_epoch_dir))
import csv
from PIL import Image


df = clustered_drawings_df
embeddings_name = "drawings"


tf_logs = f"{subclusters_dir}/tflogs"
if not os.path.isdir(tf_logs):
    os.makedirs(tf_logs)
tf_embeddings = f"{tf_logs}/embeddings-{embeddings_name}/"
if not os.path.isdir(tf_embeddings):
    os.makedirs(tf_embeddings)
metadata_fname = f"{tf_embeddings}/metadata.tsv"

with open(metadata_fname, 'w') as outfile:
    for idx in range(len(df)):
        row = df.iloc[idx]
        outfile.write(f"{row.indiv_fname}\n")
#!head {metadata_fname}

vecs_fname = f"{tf_embeddings}/feature_vecs.tsv"
with open(vecs_fname, 'w') as outfile:
    csv_writer = csv.writer(outfile, delimiter="\t")
    csv_writer.writerows([e.split(',') for e in list(df.emb_csv)])
one_square_size = int(np.ceil(np.sqrt(len(df))))
master_width = 100 * one_square_size
master_height = 100 * one_square_size
spriteimage = Image.new(
    mode='RGBA',
    size=(master_width, master_height),
    color=(0,0,0,0) # fully transparent
)
for idx in range(len(clustered_drawings_df)):
    row = clustered_drawings_df.iloc[idx]

    # TODO: rework upstream TSV handling to use non-abs paths (relative to epoch dir?)
    img_path = _shim(row.preprocessed_abs_path)

    image = Image.open(img_path).resize((100, 100))
    div, mod = divmod(idx, one_square_size)
    h_loc = 100 * div
    w_loc = 100 * mod
    spriteimage.paste(image, (w_loc, h_loc))

# important: tensorboard looks for 'jpeg' not 'jpg' when setting mimetime
spriteimage.convert('RGB').save(f'{tf_embeddings}/sprite.jpeg', transparency=0)
df = clustered_watercolors_df
embeddings_name = "watercolors"


tf_logs = f"{subclusters_dir}/tflogs"
if not os.path.isdir(tf_logs):
    os.makedirs(tf_logs)
tf_embeddings = f"{tf_logs}/embeddings-{embeddings_name}/"
if not os.path.isdir(tf_embeddings):
    os.makedirs(tf_embeddings)
metadata_fname = f"{tf_embeddings}/metadata.tsv"

with open(metadata_fname, 'w') as outfile:
    for idx in range(len(df)):
        row = df.iloc[idx]
        outfile.write(f"{row.indiv_fname}\n")
#!head {metadata_fname}

vecs_fname = f"{tf_embeddings}/feature_vecs.tsv"
with open(vecs_fname, 'w') as outfile:
    csv_writer = csv.writer(outfile, delimiter="\t")
    csv_writer.writerows([e.split(',') for e in list(df.emb_csv)])
one_square_size = int(np.ceil(np.sqrt(len(df))))
master_width = 100 * one_square_size
master_height = 100 * one_square_size
spriteimage = Image.new(
    mode='RGBA',
    size=(master_width, master_height),
    color=(0,0,0,0) # fully transparent
)
for idx in range(len(df)):
    row = df.iloc[idx]

    # TODO: rework upstream TSV handling to use non-abs paths (relative to epoch dir?)
    img_path = _shim(row.preprocessed_abs_path)

    image = Image.open(img_path).resize((100, 100))
    div, mod = divmod(idx, one_square_size)
    h_loc = 100 * div
    w_loc = 100 * mod
    spriteimage.paste(image, (w_loc, h_loc))
# important: tensorboard looks for 'jpeg' not 'jpg' when setting mimetime
spriteimage.convert('RGB').save(f'{tf_embeddings}/sprite.jpeg', transparency=0)
spriteimage

-0.6198182  -0.5714155  -0.6871516  -0.6553513  -0.6254566  -0.65341413 -0.7267234  -0.4896673  -0.7537334  -0.42715576 1.4904574   -0.72562873 -0.662768   -0.6266761  0.6913696   0.054546177 -0.5467274  -0.6520286  0.117179275 0.94330966  0.5815062   -0.55193967 -0.5446661  -0.6779002  -0.57759243 -0.4584825  -0.3818994  -0.5750927  -0.73986673 -0.45211998 -0.6141643  -0.67007643 -0.52171326 -0.4592305  1.4788612   -0.6186755  0.799198    -0.57338566 -0.17778617 0.10336053  0.093812525 0.17924881  -0.6189488  -0.6682474  2.1040225   -0.59257406 1.6553459   -0.6001429  0.5109105   0.19352531  -0.2517513  -0.66552746 -0.6482407  -0.4732165  -0.7311095  -0.6058329  -0.41851217 -0.6646769  0.19708657  -0.6859715  0.547243    -0.47478092 -0.8143768  0.6043384   -0.45561737 -0.5205148  -0.19637269 -0.67944384 -0.5505594  -0.7467994  -0.64460987 -0.46079588 -0.69849855 -0.41446227 -0.6081119  -0.6608871  0.23214674  1.7035191   -0.47101474 -0.5426106  -0.5526179  1.4402422   0.37750077  -0.4790987  0.46338224  -0.03557557 -0.577085   -0.6104121  0.31350982  0.63637584  -0.14618653 -0.61428136 1.5493102   -0.63265276 1.1587265   -0.48707122 -0.6967229  -0.59332824 1.7449057   -0.46021277 -0.63993174 -0.76553875 -0.60530424 -0.29537043 -0.4157559  -0.49291843 -0.6262977  -0.6128263  -0.5139186  -0.62823576 -0.59007895 -0.6965061  -0.5804847  -0.5246716  -0.48676312 -0.26156187 0.95585954  -0.6238418  0.9746174   -0.16204783 2.2321355   0.061881006 -0.63529575 -0.537477   -0.36832452 -0.004981935    -0.48087424 -0.43983847 -0.50105065 -0.682836   -0.66491187 -0.6797472  -0.0997439  -0.021999538    -0.71257764 1.9016916   -0.60108    -0.63299096 0.28140062  -0.7171157  -0.6375682  -0.6348774  3.2009544   -0.73214924 -0.62897754 -0.46266684 -0.37859043 -0.2960567  0.22409528  0.5868449   -0.51874804 -0.31387416 -0.19374281 0.00200814  0.7507072   1.0907284   -0.5358345  -0.07787788 -0.29217994 0.77522665  -0.55414665 -0.4696428  -0.6053274  -0.49728537 -0.55488074 -0.48824254 -0.65081346 -0.60318696 0.9660937   -0.56329983 -0.5211032  -0.639345   -0.5437213  -0.63751215 -0.34079963 -0.53909826 -0.65106076 0.20559418  -0.5073842  -0.6918073  -0.62693447 -0.49275136 -0.65002286 0.95751303  -0.3008598  -0.55680686 1.0792496   0.774848    -0.48588264 -0.71772426 -0.5829121  -0.4725283  1.023493    2.0256374   0.7612439   -0.4826959  -0.21178856 -0.5074541  2.2746863   0.47541696  1.0433946   -0.44740248 0.97176546  -0.23189563 0.23473871  -0.51704395 -0.6029111  0.027723491 -0.4995486  -0.14938274 0.1428783   -0.67159975 -0.67170393 -0.64598054 -0.5561494  -0.56587017 -0.47049776 0.22861469  -0.53188723 0.5946686   -0.61235195 -0.799528   -0.41546223 0.38230646  -0.09492311 0.33810467  1.7077615   -0.41099703 -0.6477621  -0.5815004  -0.119737774    0.3397501   -0.62445724 1.7291558   -0.59607416 1.682477    -0.666945   -0.051748693    -0.22180039 -0.7036786  -0.51746833 -0.4629042  -0.5136348  -0.44215468 -0.6816496  -0.7029415  0.23656172  -0.51101106 -0.49007544 0.1973244   -0.27828568 -0.52145916 -0.33100298 -0.5925758  -0.5095265  -0.31034473 -0.6532269  -0.7680724  1.2137315   -0.61329556 -0.6628743  0.98060465  -0.683384   0.36679065  0.2210542   0.00029248  1.2421384   1.7115324   -0.67644596 -0.20039517 0.15179002  1.2533226   0.8483739   -0.47555566 -0.588568   -0.4634904  -0.2342771  -0.17135361 0.40001863  -0.11752543 -0.517914   0.18666852  0.6040581   -0.7012117  0.3401546   -0.6307094  -0.47852555 -0.5130526  -0.50031877 0.6472873   -0.5062285  -0.6709848  1.5579097   -0.481373   -0.65100396 0.64553756  -0.71602243 0.07682127  -0.61497146 0.20312417  0.004381001 0.82400197  1.9556577   -0.6605026  -0.34786063 -0.5447662  -0.61742276 -0.6534633  -0.46339914 -0.5475428  -0.47044614 -0.6482402  -0.4996574  -0.51402634 0.36537015  -0.6583128  -0.50534487 -0.6725912  -0.22902215 -0.5561926  -0.45594007 -0.6112856  -0.15991735 -0.6999633  1.5441899   -0.6699586  0.89477324  -0.4635805  -0.58375645 -0.43037957 -0.11047655 -0.6483807  1.2622786   -0.6759538  -0.70283246 -0.5861831  -0.47019035 -0.59126043 0.18478405  -0.33273932 -0.5547614  0.3397526   1.1154402   -0.22537094 1.2848659   -0.5364493  -0.71200985 -0.5311079  -0.61185944 -0.64053524 -0.5029819  0.067080855 0.11652273  -0.5853718  -0.66932786 -0.15818292 -0.62367845 0.7262716   -0.4787713  -0.4564949  0.2395302   0.6643908   -0.25392973 -0.6761629  -0.42921948 -0.27119038 0.56590176  -0.55952317 0.36057645  1.6846812   0.057736367 -0.73007077 0.40202093  -0.5312875  -0.17738879 -0.4736769  -0.61417806 -0.4619323  -0.5301372  -0.48249543 -0.7779453  -0.19391319 -0.46213186 0.5157924   1.1046441   0.63756037  -0.61582494 -0.69205695 -0.13734245 -0.661769   -0.6900329  -0.4719818  -0.6335669  1.8876587   0.328789    -0.10699955 3.2874646   0.34532726  -0.122377336    2.7782545   0.8461142   1.3820603   -0.518363   -0.5443141  -0.6437589  -0.18877521 -0.57434005 -0.5696869  0.99117476  -0.61999637 0.26415282  -0.5783037  0.5301837   0.74880236  -0.564467   -0.624485   0.29636407  0.7744219   -0.67781055 -0.25113168 -0.43388456 0.9224855   1.8482845   -0.6042349  1.407727    1.9888644   -0.5181872  -0.56346613 0.0011875033    -0.23027515 -0.41507006 0.20999777  -0.14713013 0.890923    -0.5720823  -0.12613213 -0.68253326 -0.22734392 -0.67740345 -0.4403832  -0.5921915  -0.6676015  -0.68035614 0.48290104  -0.1582441  0.98563135  0.88531333  -0.56316394 0.12502867  -0.59465045 -0.47783846 -0.3397814  0.18029618  -0.5940082  -0.6207036  1.3442724   1.3843284   -0.64950424 -0.5894991  0.23044568  -0.6176074  -0.5994974  0.81368804  -0.16989186 -0.0060750544   -0.6400218  1.4727979   -0.22294387 2.3308163   -0.48364016 -0.4952819  -0.6471045  -0.499587   -0.5337828  -0.55979806 -0.46219397 -0.6132164  0.41023475  0.03102547  -0.6131269  -0.06623703 -0.47071788 -0.51132727 -0.22521377 -0.63688064 -0.21970803 0.9474908   -0.5632065  -0.6459049  -0.71138895 -0.39364403 -0.49940833 -0.4929026  -0.0048267245   -0.4832442  -0.5548996  -0.46071145 -0.32768083 -0.6024839  -0.6660967  0.068418026 -0.5019231  -0.6273019  -0.43346506 -0.61844975 -0.5771547  0.11831212  -0.066627264    -0.5419061  0.09847462  -0.49206764 -0.39251342
1.1073987   0.3512016   -0.73929733 -0.6553513  1.7477593   -0.65341413 0.32179314  -0.4896673  -0.1797334  -0.42715576 1.3163843   -0.72562873 -0.61387855 -0.6266761  -0.35361218 0.37611246  -0.08771938 -0.5262803  0.9261699   -0.6497359  -0.64108604 -0.55193967 -0.5446661  -0.6779002  -0.32559583 -0.6690378  -0.4518578  0.51828945  -0.4238917  -0.45211998 -0.16527033 -0.67007643 1.0136865   -0.4592305  -0.600525   -0.060860336    -0.19446748 -0.24127388 2.2282472   1.8003348   -0.61049384 0.39795423  0.2626753   -0.6682474  -0.75794554 -0.59257406 0.80257696  -0.6001429  -0.651804   -0.028328001    0.53641737  -0.6501347  -0.6482407  0.3295404   -0.7311095  0.7769913   -0.47955433 0.029548049 -0.69804686 -0.6859715  -0.7994243  -0.37796497 0.2770447   -0.5895213  -0.45400283 -0.3217132  -0.69643825 -0.50339925 -0.28047013 -0.7467994  -0.64460987 -0.46079588 0.9519163   -0.692441   -0.6081119  -0.6608871  -0.68580395 0.38670623  -0.39434192 -0.2689916  -0.5526179  3.0988364   0.6767974   0.5479597   -0.64532113 -0.79415584 -0.577085   -0.6104121  -0.21005714 0.9130197   0.5350442   0.02642548  -0.7075963  -0.63265276 1.0843167   -0.14746365 0.37673205  -0.73169684 2.9166942   -0.68441904 -0.63993174 -0.76553875 1.35689 -0.2936016  1.1953154   -0.49291843 2.4941373   0.5234068   -0.5519926  -0.62823576 -0.27636325 -0.6965061  -0.5804847  -0.3484809  -0.23164913 0.3351344   1.9399989   -0.6238418  2.4131138   0.56315875  -0.6619385  0.7226322   3.0301828   -0.537477   -0.5148479  1.2457952   -0.035156786    0.0038073957    -0.50105065 -0.22542143 0.79507375  -0.6797472  0.4369036   -0.6556923  -0.71257764 0.842523    1.168153    -0.56751305 1.2122676   -0.7171157  -0.6375682  -0.6348774  -0.5785726  -0.73214924 0.50270844  -0.46266684 1.4489454   -0.38700646 -0.6088965  1.3957318   -0.51874804 2.1159246   -0.6530593  -0.024851084    -0.59888333 0.18911964  -0.5358345  -0.6222166  0.19376683  0.014755368 -0.55414665 -0.4696428  -0.6053274  0.22089148  -0.55488074 0.24218276  -0.65081346 -0.60318696 0.659924    -0.56329983 -0.08825785 -0.639345   -0.027083099    -0.63751215 1.481066    -0.53909826 0.40229827  -0.68776333 0.7223346   -0.6918073  -0.62693447 0.049732804 -0.65002286 -0.21168253 0.009727299 -0.55680686 -0.6456044  -0.25574428 -0.48588264 -0.71772426 -0.7531246  -0.4725283  0.85692835  -0.30708647 2.8895497   -0.4826959  0.65924776  -0.5074541  -0.39449766 -0.06298071 1.1019258   -0.44740248 0.26116616  -0.41378984 -0.64094496 0.34735852  0.88798976  -0.6500554  -0.4995486  1.0063698   -0.7774646  -0.67159975 -0.67170393 -0.47006562 -0.5561494  -0.094100624    -0.47049776 -0.7183699  -0.09890795 -0.6599151  -0.61235195 -0.799528   -0.5958991  -0.5961882  -0.53419805 -0.17983526 -0.078457534    1.0881107   -0.6477621  -0.5815004  1.790916    -0.62075394 -0.62445724 1.0470514   1.1369569   2.796564    -0.67020726 -0.58545256 2.2493072   0.528484    0.5199286   -0.5639156  -0.5136348  -0.44215468 0.22048092  -0.7029415  -0.6352208  -0.19745034 0.6001003   -0.64529026 -0.631587   -0.52145916 -0.68476003 -0.5925758  0.2412672   -0.29952493 0.033296943 -0.7680724  1.2963097   0.23716837  -0.6628743  0.0137838125    -0.683384   3.0933921   -0.27605537 -0.24247721 -0.24648625 -0.6831762  -0.67644596 0.09041631  -0.0870235  0.8275259   -0.63977027 -0.47555566 -0.588568   -0.44164154 -0.5791197  -0.5512841  1.861238    -0.060551345    -0.056833237    1.0406823   -0.62749094 -0.5422507  0.41750175  0.25172496  -0.47852555 -0.36199635 -0.40971822 0.1049726   -0.5062285  -0.6709848  -0.69386715 -0.481373   -0.65100396 0.28651512  -0.71602243 -0.6275605  -0.3905087  -0.6981776  0.3271085   1.391211    -0.8038508  0.70832914  -0.73018485 -0.5447662  -0.61742276 -0.6534633  0.5830945   -0.004030347    -0.47044614 -0.6482402  0.3220634   1.077738    -0.62476414 -0.6583128  -0.50534487 0.18389475  0.9850725   -0.5561926  0.07914758  1.916964    -0.62447065 0.62855464  0.95698535  1.1300554   -0.61644304 -0.4635805  1.5010269   -0.43037957 -0.6291859  -0.6483807  0.186557    0.036331773 0.920956    -0.5861831  -0.47019035 -0.59126043 0.46994698  -0.6811046  -0.5547614  1.1410706   -0.42045763 1.7019651   -0.5869171  -0.5364493  -0.71200985 -0.5311079  3.415893    2.6726599   -0.44865486 2.536608    -0.032728553    -0.5780154  0.44250727  -0.16620338 -0.62367845 0.85426545  -0.4787713  0.17900649  2.1816025   0.32237065  0.17274886  -0.5489194  -0.6687816  -0.62331444 0.527226    -0.55952317 -0.6330787  -0.6756778  1.0632998   -0.73007077 0.8655244   -0.5563068  -0.7210105  -0.4736769  0.07629979  -0.4619323  -0.5301372  -0.5188687  -0.7779453  -0.5511636  0.06920338  0.644273    1.2033467   -0.7352437  -0.61582494 -0.69205695 0.84463143  0.6501573   -0.6900329  -0.06875408 -0.6335669  -0.7927979  -0.65140545 -0.54773664 0.25930458  1.2992175   0.5042671   1.7256842   0.7275955   -0.6175546  -0.4735519  -0.31053644 -0.6437589  -0.6362093  -0.4980286  -0.05924201 1.0999625   -0.61999637 0.50024706  -0.5783037  -0.63265276 0.9367605   1.0552256   -0.624485   1.5743426   -0.6635577  -0.3704734  1.4905856   1.5606068   0.51483727  -0.49503195 -0.6219417  1.236131    -0.7140479  -0.19985345 1.6453497   -0.5921287  -0.25122356 -0.5988868  -0.66452193 -0.70562786 1.1116987   -0.4090636  0.71391904  -0.68253326 -0.55096567 0.29692745  0.021855652 0.80976707  -0.6676015  -0.68035614 0.7990009   0.3743217   1.7308805   -0.39844143 -0.20389384 -0.7522869  -0.59465045 -0.47783846 -0.189868   -0.67154586 -0.34900972 -0.6207036  0.6796166   -0.6108009  -0.64950424 -0.5894991  -0.57587796 -0.6176074  -0.5994974  -0.6213416  -0.6484237  0.42586687  -0.6400218  0.07455218  -0.7007099  -0.28212306 -0.48364016 -0.4952819  -0.6471045  -0.009717941    0.009052992 -0.55979806 -0.46219397 -0.6132164  -0.5378847  1.5603629   1.9842507   -0.6269447  1.0171154   -0.51132727 -0.63025486 -0.63688064 0.3612367   -0.59816146 -0.5632065  -0.6459049  -0.71138895 -0.5816399  -0.49940833 -0.4929026  -0.7398169  -0.4832442  -0.5548996  0.30212536  0.5373781   -0.6024839  1.0492686   0.3141206   -0.5019231  0.655814    -0.10403547 -0.61844975 0.35397965  -0.028526545    0.6035874   0.1292668   -0.65107447 -0.49206764 0.32716987
1.3776337   -0.5714155  -0.2587232  -0.6553513  1.3532461   -0.65341413 -0.7267234  -0.4896673  -0.7537334  -0.25077122 0.050992966 -0.72562873 0.1406585   -0.6266761  1.0653365   0.96198404  -0.5467274  0.30261123  0.6327609   -0.6497359  -0.64108604 -0.28434965 -0.35651857 -0.6779002  -0.28245115 -0.6690378  -0.4518578  0.030088186 -0.73986673 -0.18931103 -0.041653574    -0.67007643 -0.52171326 -0.29881266 -0.600525   -0.23231092 -0.73933    -0.57338566 -0.65925777 1.8833572   -0.61049384 0.44560486  0.14075392  -0.6682474  -0.75794554 -0.59257406 -0.07511848 -0.6001429  -0.651804   -0.5315948  0.023931146 0.723624    -0.6482407  -0.13593137 -0.7311095  0.59529644  -0.47955433 -0.6646769  -0.69804686 -0.49380377 -0.47234848 -0.47478092 0.55306476  0.79116476  -0.5287406  -0.06111142 -0.69643825 -0.67944384 -0.59154034 -0.7467994  -0.64460987 -0.46079588 -0.69849855 -0.692441   -0.6081119  -0.6608871  0.71966714  -0.8218076  -0.47101474 -0.5426106  -0.5526179  1.6324786   -0.51120186 -0.4790987  -0.64532113 0.028811932 -0.577085   -0.6104121  0.5281577   -0.039097607    -0.71985745 -0.61428136 1.0950723   -0.63265276 1.7796504   -0.50889117 -0.6967229  -0.73169684 0.13936383  -0.68441904 -0.63993174 -0.76553875 -0.0690189  -0.55279887 1.3697383   -0.49291843 0.28409517  -0.6128263  -0.5519926  -0.013880551    0.31387687  -0.6965061  0.19958514  -0.46852648 1.6816958   0.008445114 2.8924024   -0.6238418  1.8272073   0.17341909  1.2846789   -0.58417743 0.7973143   -0.537477   -0.5148479  0.34587818  -0.48087424 0.0909127   -0.4703402  -0.682836   0.4384197   -0.6797472  1.0529947   -0.6556923  -0.71257764 -0.02925086 -0.60108    -0.63299096 0.49842006  -0.7171157  -0.6375682  -0.6348774  0.12574118  -0.73214924 0.3981874   -0.46266684 0.99739885  -0.5647883  -0.6088965  1.018522    -0.51874804 3.6447055   0.088588715 -0.16168177 0.4316172   -0.70976746 -0.5358345  -0.6222166  -0.58524024 -0.56878453 -0.26676238 -0.4696428  -0.3541621  -0.21271363 -0.18079734 -0.48824254 -0.65081346 -0.60318696 -0.11717546 -0.56329983 0.5343043   -0.639345   -0.40191436 -0.63751215 -0.60767806 -0.53909826 -0.0073986053   0.44983995  0.8738009   -0.6918073  -0.62693447 -0.45662212 -0.65002286 -0.6848493  1.9475645   -0.55680686 -0.6456044  -0.70373756 0.17998844  -0.107574165    -0.7531246  -0.4725283  -0.6788409  0.4517892   1.121084    -0.4826959  0.75132203  -0.5074541  0.86952305  2.1392052   1.6924658   -0.44740248 -0.5509034  -0.493435   1.1955667   -0.51704395 -0.6029111  -0.6500554  -0.4995486  -0.52000505 -0.025108695    -0.058374166    -0.43562713 -0.64598054 -0.5561494  -0.56587017 0.5579486   0.36710173  -0.53188723 -0.6599151  -0.43994385 -0.799528   -0.5958991  0.87169635  -0.06929764 -0.47800088 0.97562116  -0.23208794 -0.6477621  -0.5815004  2.277323    -0.62075394 -0.19751352 -0.5734308  2.0764236   2.1207898   -0.67020726 -0.58545256 -0.5507498  -0.39684898 0.14172232  -0.12031591 -0.5136348  -0.44215468 -0.6816496  0.9461835   0.0978117   -0.51101106 0.65664434  -0.2258716  -0.631587   -0.52145916 -0.10124171 -0.5925758  -0.5095265  -0.5200247  -0.6532269  -0.7680724  -0.5927339  -0.61329556 -0.6628743  -0.346667   -0.683384   0.8354152   -0.21816382 1.9323242   -0.31244957 -0.6831762  -0.67644596 0.042757213 -0.5247881  0.6151841   -0.63977027 -0.47555566 -0.588568   -0.4634904  -0.5791197  -0.5512841  1.633877    -0.0636791  -0.517914   0.34298027  -0.62749094 -0.7012117  0.99376136  -0.6307094  -0.47852555 -0.5130526  -0.4842003  -0.6389534  -0.5062285  -0.41510683 1.544627    -0.481373   -0.65100396 -0.664848   0.87122625  -0.6275605  0.12040424  -0.6981776  -0.0033897161   -0.5311089  -0.7548808  -0.6605026  -0.73018485 0.68097436  -0.61742276 -0.6534633  0.09401286  -0.5475428  -0.47044614 -0.6482402  -0.031416506    -0.51402634 -0.4438041  0.6428673   -0.50534487 -0.6725912  -0.5148951  -0.5561926  -0.45594007 -0.6112856  -0.62447065 -0.6999633  0.5318935   -0.6699586  0.8117211   -0.15828401 0.13347578  -0.43037957 -0.6291859  0.63440746  0.9724966   1.8258135   0.77202344  -0.27735382 -0.47019035 -0.59126043 0.76534235  -0.6811046  -0.5547614  1.9455194   0.36043268  0.8312363   -0.6157184  -0.5364493  0.056869686 -0.5311079  -0.61185944 0.5615393   -0.7814901  1.793325    -0.70669377 -0.5853718  -0.66932786 -0.5585858  -0.62367845 -0.10432693 -0.4787713  -0.4564949  0.05945313  -0.20767814 -0.35012648 -0.122913   -0.6687816  -0.6420673  1.2881483   1.313288    -0.6330787  -0.6756778  0.2662569   -0.73007077 0.9952432   -0.5563068  -0.7210105  -0.4736769  -0.61417806 -0.4619323  -0.5301372  -0.5188687  -0.32171673 -0.5511636  -0.46213186 -0.36939648 0.50612587  -0.40164778 -0.61582494 0.581302    3.238738    -0.661769   -0.6900329  -0.4719818  -0.6335669  -0.39032048 -0.65140545 -0.19066495 0.29106563  1.6273549   -0.6868099  0.459081    -0.61867577 -0.6175546  0.27739727  0.015430629 -0.6437589  -0.6362093  -0.050485313    -0.06652564 0.57567877  -0.61999637 0.79358846  -0.5783037  -0.63265276 -0.30958584 0.022988081 -0.624485   0.19309413  1.8246684   0.1240108   0.8768455   1.0637047   2.2979016   -0.36150196 0.15229255  0.30510563  -0.24800098 -0.5181872  -0.084780484    -0.6898156  -0.49261582 -0.5988868  -0.66452193 -0.70562786 0.122772336 -0.3561841  -0.4018598  -0.68253326 -0.55096567 -0.05666709 -0.4313164  0.58647746  -0.289331   -0.68035614 0.75362533  -0.64309394 1.5269258   1.9408898   -0.56316394 -0.7522869  0.29567719  -0.47783846 -0.35840648 -0.67154586 -0.5940082  0.67609406  1.6258705   -0.683239   0.33839643  -0.5894991  -0.15334094 0.7860455   -0.5994974  -0.5924538  -0.6484237  -0.49718592 -0.6400218  0.68910927  -0.7007099  2.23881 -0.48364016 -0.4952819  0.06308985  -0.499587   -0.5337828  -0.55979806 -0.4478871  -0.6132164  -0.5378847  0.03778881  0.43626726  -0.6269447  2.814978    -0.51132727 -0.26461604 -0.63688064 1.5497985   -0.59816146 0.49051863  -0.6459049  -0.54030514 -0.5816399  -0.49940833 -0.00710088 -0.7398169  -0.4832442  1.0332335   -0.4584871  -0.40195614 -0.6024839  0.46805418  -0.7452056  -0.5019231  -0.6273019  -0.53396827 -0.61844975 -0.5771547  -0.31358904 2.3046937   0.35861295  -0.65107447 -0.4748875  0.19909522
tf_embeddings
'/Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/04_SUBCLUSTERS/tflogs/embeddings/'
<tf.Variable 'Variable:0' shape=(1923, 512) dtype=float32, numpy=
array([[-0.6198182 , -0.5714155 , -0.6871516 , ...,  0.09847462,
        -0.49206764, -0.39251342],
       [ 1.1073987 ,  0.3512016 , -0.73929733, ..., -0.65107447,
        -0.49206764,  0.32716987],
       [ 1.3776337 , -0.5714155 , -0.2587232 , ..., -0.65107447,
        -0.4748875 ,  0.19909522],
       ...,
       [ 1.637738  , -0.5714155 , -0.73929733, ..., -0.65107447,
        -0.13707963,  0.20361403],
       [ 4.6632442 ,  0.29762375, -0.73929733, ...,  0.01897907,
        -0.49206764, -0.39251342],
       [ 0.71924925,  0.20198071,  1.2284024 , ..., -0.22790754,
         0.7221783 , -0.08996937]], dtype=float32)>
weights

# weights = tf.Variable(model.layers[0].get_weights()[0][1:])
<tf.Variable 'Variable:0' shape=(1923, 512) dtype=float32, numpy=
array([[-0.6198182 , -0.5714155 , -0.6871516 , ...,  0.09847462,
        -0.49206764, -0.39251342],
       [ 1.1073987 ,  0.3512016 , -0.73929733, ..., -0.65107447,
        -0.49206764,  0.32716987],
       [ 1.3776337 , -0.5714155 , -0.2587232 , ..., -0.65107447,
        -0.4748875 ,  0.19909522],
       ...,
       [ 1.637738  , -0.5714155 , -0.73929733, ..., -0.65107447,
        -0.13707963,  0.20361403],
       [ 4.6632442 ,  0.29762375, -0.73929733, ...,  0.01897907,
        -0.49206764, -0.39251342],
       [ 0.71924925,  0.20198071,  1.2284024 , ..., -0.22790754,
         0.7221783 , -0.08996937]], dtype=float32)>
# Create a checkpoint from embedding, the filename and key are the
# name of the tensor.
checkpoint = tf.train.Checkpoint(embedding=weights)
checkpoint.save(os.path.join(tf_embeddings, "embedding.ckpt"))

checkpoint.save(os.path.join(tf_logs, "embedding.ckpt"))
'/Users/al/Dropbox/2-Areas/2-Sketchbooks/datasets/full-v2/04_SUBCLUSTERS/tflogs/embedding.ckpt-2'
# Set up config.
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
# The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE`.
embedding.tensor_name = "embedding/Variable:0"
embedding.metadata_path = 'metadata.tsv'
projector.visualize_embeddings(tf_embeddings, config)
Launching TensorBoard...