Notebook Three | Report | Repository

Tags vectorisation

Andrea Leone
University of Trento
January 2022

import project 

import numpy  as np
import pandas as pd
import collections

import sklearn
import sklearn.decomposition
import matplotlib.pyplot as plt

from tqdm.notebook import tqdm

plt.rcParams['figure.figsize'] = [ 20.0, 9.0 ]
plt.rcParams['figure.dpi']     =  120

Analyse the tag distribution in the database: extract all tags of each talk and store them in a set

records = project.sql_query("SELECT tags, slug FROM talks ORDER BY slug ASC;") 
tags    = collections.Counter()

for record in records : tags.update( record[0] )

len(records), len(tags)

(5363, 347)

Get the frequency of each tag in the set

tags.most_common()[:15]

[('science', 1218),
 ('technology', 1177),
 ('ted-ed', 819),
 ('culture', 782),
 ('animation', 744),
 ('education', 704),
 ('society', 697),
 ('tedx', 681),
 ('social change', 645),
 ('global issues', 631),
 ('design', 580),
 ('business', 573),
 ('health', 558),
 ('history', 557),
 ('innovation', 479)]

Plot the tag frequency distribution

plt.stem( 
    sorted( [y for x, y in tags.most_common()], reverse=True )[:100]
); plt.show()

Tags are manifold: select a bunch and check the frequency

tag_frequency = dict(tags.most_common())

tag_names = [ 
    'life', 'health', 'happiness', 'identity', 'love', 'nature', 'money', 'society', 'culture',
    'global issues', 'social change', 'sustainability', 'history', 'biology', 'humanity',
    'art', 'literature', 'creativity', 'engineering', 'journalism', 'communication',
    'education', 'business', 'democracy', 'ethics', 'poverty',
    'food', 'climate change', 'coronavirus', 'immigration', 'activism', 'goals',
    'future', 'science', 'innovation', 'technology', 'internet', 'software', 'ai'
]

tag_frequency = dict(tags.most_common())
tag_counts    = [ tag_frequency[tag] for tag in tag_names ]

plt.stem( 
    [ y for x,y in sorted( dict(zip(tag_names, tag_counts)).items(), key=lambda x : x[1], reverse=True ) ]
); plt.show()

sorted( dict(zip(tag_names, tag_counts)).items(), key=lambda x : x[1], reverse=True )[:15]

[('science', 1218),
 ('technology', 1177),
 ('culture', 782),
 ('education', 704),
 ('society', 697),
 ('social change', 645),
 ('global issues', 631),
 ('business', 573),
 ('health', 558),
 ('history', 557),
 ('innovation', 479),
 ('humanity', 476),
 ('biology', 458),
 ('communication', 420),
 ('art', 401)]

Create a dictioary with three main categories, each one collecting the tags that describe or concern it.

tag_dict = { 
    
    'science and innovation': [
        'biology', 'technology', 'ai', 'future', 'medicine', 'brain', 'math',
        'science', 'innovation', 'invention', 'engineering', 'medical research',
        'internet', 'space', 'media', 'computers', 'data', 'disease', 'demo',
        'evolution', 'physics', 'exploration', 'mental health', 'neuroscience',
        'architecture', 'illness', 'universe', 'visualizations', 'law', 'software',
        'genetics', 'virus', 'robots', 'biotech', 'cancer', 'aging', 'cognitive science',
        'astronomy', 'infrastructure', 'dna', 'algorithm', 'memory', 'youth', 'product design',
        'insects', 'ethics', 'coronavirus', 'machine learning', 'investing',
        'chemistry', 'code', 'anthropology', 'countdown', 'microbiology',
        'best of the web', 'curiosity', 'statistics', 'online privacy', 'vaccines',
        'anthropocene', 'time', 'sight', 'industrial design', 'discovery', 'solar system',
        'flight', 'sound', 'bioethics', 'surveillance', 'audacious project',
        'nanotechnology', 'encryption', 'maps', 'biomimicry', 'drones', 'virtual reality',
        'surgery', 'heart', 'quantum', 'telescopes', 'synthetic biology', 'big bang',
        'aliens', 'medical imaging', 'botany', 'dark matter', 'paleontology', 'mars',
        'cyber security', 'exercise', 'television', 'toys', 'astrobiology', 'nuclear energy',
        'geology', 'driverless cars', 'rocket science', 'blockchain', 'string theory',
        'augmented reality', 'cryptocurrency', 'crispr'
    ],
    
    'culture and society': [
        'education', 'history', 'humanity', 'art', 'communication', 'entertainment',
        'community', 'activism', 'psychology', 'politics', 'women', 'identity',
        'social change', 'health', 'journalism', 'storytelling', 'depression', 'feminism',
        'government', 'equality', 'public health', 'race', 'gender', 'refugees',
        'humor', 'language', 'violence', 'family', 'diversity', 'self', 'religion',
        'society', 'culture', 'democracy', 'happiness', 'books', 'relationships',
        'philosophy', 'poetry', 'parenting', 'social media', 'death', 'literature',
        'emotions', 'beauty', 'ancient world', 'sex', 'love', 'crime', 'public space',
        'justice system', 'compassion', 'empathy', 'comedy', 'disability',
        'consciousness', 'life', 'lgbtqia+', 'dance', 'drugs', 'fear', 'philanthropy',
        'gaming', 'terrorism', 'prison', 'corruption', 'personality', 'sociology',
        'illusion', 'spoken word', 'aids', 'sexual violence', 'trust', 'slavery',
        'transgender', 'pain', 'library', 'protest', 'painting', 'meditation',
        'inclusion', 'addiction', 'museums', 'friendship', 'mindfulness', 'vulnerability',
        'human rights', 'body language', 'public speaking', 'bullying', 'suicide',
        'atheism', 'autism spectrum disorder', 'grammar'
    ],
    
    'economy and environment': [
        'business', 'design', 'creativity', 'economics', 'personal growth', 'performance',
        'environment', 'climate change', 'nature', 'war', 'sustainability', 'animals',
        'global issues', 'human body', 'kids', 'united states', 'india', 
        'music', 'work', 'africa', 'leadership', 'cities', 'ocean', 'food', 'writing',
        'collaboration', 'health care', 'energy', 'photography', 'decision-making', 'policy',
        'entrepreneur', 'potential', 'motivation', 'film', 'biodiversity',
        'international development', 'teaching', 'poverty', 'pollution', 'money', 'finance',
        'water', 'success', 'ecology', 'transportation', 'agriculture', 'sports', 'goals',
        'fish', 'bacteria', 'marine biology', 'planets', 'resources', 'urban planning',
        'conservation', 'plants', 'electricity', 'work-life balance', 'renewable energy',
        'natural disaster', 'pandemic', 'natural resources', 'farming', 'marketing',
        'productivity', 'ux design', 'consumerism', 'fashion', 'behavioral economics',
        'military', 'sleep', 'solar energy', 'trees', 'theater', 'weather', 'archaeology',
        'biosphere', 'capitalism', 'manufacturing', 'gardening', 'travel', 'mission blue',
        'birds', 'plastic', 'graphic design', 'fossil fuels', 'europe', 'women in business',
        'middle east', 'china', 'asia', 'indigenous peoples', 'immigration', 'south america',
        'international relations', 'sun', 'moon', 'bees', 'wind energy', 'rivers',
        'glaciers', 'coral reefs', 'microbes', 'deextinction'
    ],
    
    'extra': [
        'tedx', 'ted-ed', 'ted connects', 'ted membership', 'ted fellows', 'ted prize',
        'ted residency', 'ted membership', 'interview', 'podcast', 'crowdsourcing'
    ]
    
}

Assign each record to one of the three categories

# careful! 

for record in tqdm( records ):

    tags = record[0]
    slug = record[1]

    dct = {
        'science and innovation'  : 0,
        'culture and society'     : 0,
        'economy and environment' : 0
    }

    for tag in tags:
        for label in list(dct.keys()):
            if tag in tag_dict[label]:
                dct[label] += 1
    
    value = list( dct.values() ).index( max( dct.values() ) )
    
    project.sql_commit("UPDATE talks SET category='{0}' WHERE slug='{1}'".format(value, slug))

  0%|          | 0/5363 [00:00<?, ?it/s]

Explore the data distribution according to the three categories.

records = project.sql_query(""" 
    SELECT category, slug FROM talks
    WHERE vector IS NOT NULL
    ORDER BY slug ASC;
""")

categories, slugs = project.unzip_array( records )
collections.Counter(categories)

Counter({1: 1847, 2: 1244, 0: 1619})

Data seems quite balanced. Now, load and compress the vectors.

pca = lambda X : sklearn.decomposition.PCA(n_components=3).fit_transform(X) 

transcript_vectors = project.sql_query("""
    SELECT vector, vector_trf FROM talks
    WHERE vector IS NOT NULL
    ORDER BY slug ASC;
""")

vectors, vectors_trf   = project.unzip_array( transcript_vectors )

compressed_vectors     = pca( vectors )
#compressed_vectors_trf = pca( vectors_trf )

Arrange the data for a 3D-scatterplot and see the result

scatter_data   = list( zip(categories, compressed_vectors) ) 
scatter_colors = [ 'deepskyblue', 'crimson', 'gold' ]

project.scatterplot3 (scatter_data, scatter_colors, xl=(-0.85,+0.45), yl=(-0.45,+3.45), zl=(-0.75,+0.35))