Commit 248264fa authored by Marcel Henrik Schubert's avatar Marcel Henrik Schubert
Browse files

added all recent changes and scripts

parent 628bde6d
This diff is collapsed.
This diff is collapsed.
......@@ -3,12 +3,14 @@ import sys
import gc
import json
import jsonlines
import pickle
import pandas as pd
import tables
#import tables
from scipy import sparse
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, HashingVectorizer
def store_sparse_mat(M, name, filename='store.h5'):
'''def store_sparse_mat(M, name, filename='store.h5'):
Store a csr matrix in HDF5
......@@ -39,9 +41,9 @@ def store_sparse_mat(M, name, filename='store.h5'):
arr = np.array(getattr(M, attribute))
atom = tables.Atom.from_dtype(arr.dtype)
ds = f.create_carray(f.root, full_name, atom, arr.shape)
ds[:] = arr
ds[:] = arr'''
def load_sparse_mat(name, filename='store.h5'):
'''def load_sparse_mat(name, filename='store.h5'):
Load a csr matrix from HDF5
......@@ -67,7 +69,7 @@ def load_sparse_mat(name, filename='store.h5'):
# construct sparse matrix
M = sparse.csr_matrix(tuple(attributes[:3]), shape=attributes[3])
return M
return M'''
def split_path_unix_win(path):
#make windows-unix problem go away
......@@ -84,7 +86,8 @@ def batch_read(fileobj, batchsize, to_json=True):
line = fileobj.readline()
if line and to_json:
line = jsonize(line)
line['uID'] = make_uids(line, 'tweetIDs')
line = make_uids(line, 'tweetIDs', secondary='tweetID')
elif line:
......@@ -105,37 +108,295 @@ def jsonize(line):
return line
def make_uids(dic, key):
dic[key] = '|'.join(dic[key])
def make_uids(dic, key, secondary='tweetID'):
if key not in dic.keys():
key = secondary
dic['uID'] = '|'.join(dic[key])
return dic
def select_uids(df, ids):
df = df.loc[df['uID'].isin(ids),:]
return df
def pandas_to_ndjson(df:pd.DataFrame, f):
df = df.to_records()
writer = jsonlines.Writer(df)
df = df.to_dict(orient='records')
writer = jsonlines.Writer(f)
print('wrote subset to file...')
def make_save_dirs(path):
# create files for later saving
path = split_path_unix_win(path)
ind = path.index('preprocessed')
os.makedirs(os.path.join(*(path[0:ind]+ ['models'] + path[(ind+1):])))
model_dir = os.path.join(*(path[0:ind]+ ['models'] + path[(ind+1):]))
os.makedirs(model_dir, exist_ok=True)
print('made models directory with subdirs...')
return model_dir
def identity_prepr(text):
return text
def reset_num(num:pd.DataFrame, fix:pd.DataFrame):
num = num.sort_values(by=['uID'])
fix = fix.sort_values(by=['uid']).copy(deep=True)
#fix emoji count
num.loc[num.isin(fix.uID), 'emojis_num'] = fix.emoji.apply(len)
num.loc[num.isin(fix.uID), 'emotic_num'] = fix.emoticon.apply(len)
return num
def identity_tokenizer(text):
return text
def identity_analyzer(text):
return text
\ No newline at end of file
return text
def zero_columns(M, columns):
diag = sparse.eye(M.shape[1]).tolil()
for c in columns:
diag[c, c] = 0
def make_vocab(dict, path):
with open(path, 'w', encoding='utf-8') as json_file:
json.dump(dict, json_file, ensure_ascii=False)
print('saved vocab to disk...')
def get_vocab(path):
with open(path, 'r', encoding='utf-8') as json_file:
vocab = json.load(json_file)
return vocab
def update_vocab(vocab, lines:pd.DataFrame, key):
for el in lines[key]:
subvocab = {}
#basically make a set here - we do not want total frequency but doc frequency (i.e. in how many docs does it appear not how often does it appear overall
for token in el:
subvocab[token] = subvocab.get(token, 0) + 1
for token in subvocab.keys():
vocab[token] = vocab.get(token, 0) +1
return vocab
def reduce_vocab(vocab, cutoff):
ind = 0
new = {}
for key in vocab.keys():
if vocab[key] >=cutoff:
new[key] = ind
ind +=1
return new, ind
def vocab_maker(vocab):
ind = 0
new = {}
for key in vocab.keys():
new[key] = ind
ind +=1
return new, ind
def construct_tdm(docs, vocab, maxcol):
indptr = [0]
indices = []
data = []
exception_counter = 0
term_counter = 0
for d in docs:
for term in d:
term_counter +=1
index = vocab.get(term, 'exception') #if term does not appear often enough
if index == 'exception':
exception_counter +=1
#make it so that we have enough columns
print('went over {} terms'.format(term_counter))
print('of these {} are not in dict'.format(exception_counter))
return sparse.csr_matrix((data, indices, indptr), dtype=int)
def find_variance_cutoff(pca, dat, cutoff=0.99, tol = 0.005, start = 100):
#recurisve cutoff search (divide and conquer)
print('make pca with {} components'.format(start))
pca = pca.set_params(n_components=start)
#pca = pca.set_params(batch_size = round(start*3))
pca =
diff = np.abs(pca.explained_variance_ratio_.sum() - cutoff)
print('Difference to cutoff is {}'.format(diff))
if diff < tol:
print('Use {} components. That is {} of num_features'.format(start, round(start/dat.shape[1], 2)))
return pca
#else we continue search recursively
if pca.explained_variance_ratio_.sum() > cutoff:
start = round(3*start/4)
return find_variance_cutoff(pca, dat, cutoff, tol, start = start)
start = start + round((dat.shape[1]-start)/2)
return find_variance_cutoff(pca, dat, cutoff, tol, start=start)
def make_gram_dict(path:str):
gram_dict = {}
path = os.path.join(*split_path_unix_win(path))
for dir in os.listdir(path):
if dir not in ['vectors', 'process', 'age_ids', 'gender_ids']:
direct = os.path.join(path, dir)
if os.path.isdir(direct):
#make key char, num, etc.
gram_dict[dir] = {}
for tweetLen in os.listdir(direct):
subdirect = os.path.join(direct, tweetLen)
if os.path.isdir(subdirect):
#make key tweetlen 500, 250, 100
gram_dict[dir][str(tweetLen)] = {}
for target in os.listdir(subdirect):
target_direct = os.path.join(subdirect, target)
if os.path.isdir(target_direct):
#make target key
gram_dict[dir][str(tweetLen)][target] = {}
for subset in os.listdir(target_direct):
subset_dir = os.path.join(target_direct, subset)
if os.path.isdir(subset_dir):
#make subset key, i.e. train, test, val
gram_dict[dir][str(tweetLen)][target][subset] = {}
#now we only have files (sometimes wtih and sometimes without grams
files = os.listdir(subset_dir)
for file in files:
##now we have to go over the number of authors in each subset
authors = file.split('_')[-2]
#make or keep key with dictionary
gram_dict[dir][str(tweetLen)][target][subset][authors] = gram_dict[dir][str(tweetLen)][target][subset].get(authors, {})
if '_grams_' in file:
grams = file.split('_')[-4]
gram_dict[dir][str(tweetLen)][target][subset][authors][grams] = os.path.join(subset_dir, file)
gram_dict[dir][str(tweetLen)][target][subset][authors]['no_gram'] = os.path.join(subset_dir, file)
gram_dict[dir][str(tweetLen)][target][subset][authors]['ids'] = os.path.join(path, '{}_ids'.format(target),
subset, '{}_ids_{}_{}_{}_authors_balanced.json'.format(subset,
return gram_dict
def assess_equal(dat1, ids1, dat2, ids2):
sym = set(ids1).symmetric_difference(set(ids2))
if len(ids1) > len(ids2):
assert len(set(ids2) - set(ids1)) == 0
print('data1 is ({}) larger than dat2 ({})...'.format(dat1.shape, dat2.shape))
dat1 = assess_order(np.array(ids2), dat1, np.array(ids1))
ids1 = ids2
assert len(set(ids1) - set(ids2)) == 0
print('data2 is ({}) larger than dat1 ({})...'.format(dat2.shape, dat1.shape))
dat2 = assess_order(np.array(ids1), dat2, np.array(ids2))
ids2 = ids1
return dat1, ids1,dat2, ids2
def assess_order(sorter,data, data_ids):
if type(sorter) == type({}):
sorter = sorter[list(sorter.keys())[0]]
sorter = np.array(sorter).flatten()
assert type(data_ids) == type(np.array([]))
d = {k:v for v,k in enumerate(data_ids)}
sort_ids = [d[el] for el in sorter] #order we want the ids in
#d = {k:v for v,k in enumerate(sorter)}
#sort_ids = [d[el] for el in data_ids]
return data[sort_ids, :] #slice the array in correct order
def load_make_vectorizers(target, path, featuretyp, minTw_gram_auth, hash):
vectorizer_dic = {}
minTW = minTw_gram_auth[0]
numAuthors = minTw_gram_auth[2]
path_vocab = os.path.join(*split_path_unix_win(path),
featuretyp, str(minTW), 'vocab', target)
for gram in sorted(minTw_gram_auth[1]):
vectorizer_dic[str(gram)] = {}
if not hash:
path_vocab_red = os.path.join(path_vocab,
'{}_grams_{}_{}_{}_authors_reduced.vocab'.format(featuretyp, gram, minTW,
vocab = get_vocab(path_vocab_red)
vectorizer_dic[str(gram)]['vect'] = CountVectorizer(vocabulary=vocab, analyzer=identity_analyzer)
vectorizer_dic[str(gram)]['vocab'] = vocab
path_colids = os.join.path(path_vocab,
'{}_grams_{}_{}_{}_authors_reduced.colids'.format(featuretyp, gram, minTW,
vectorizer_dic[str(gram)]['vect'] = HashingVectorizer(analyzer=identity_analyzer)
with open(path_colids, 'rb') as p:
colinds = pickle.load(p)
vectorizer_dic[str(gram)]['colinds'] = colinds
return vectorizer_dic
def make_prediction_dir(path, featuretyp, minTweet, target):
path = split_path_unix_win(path)
ind = path.index('preprocessed')
path[ind] = 'predictions'
path = os.path.join(*path, featuretyp, minTweet,target)
return path
def y_categories_target_key(target):
if 'age' in target:
target_key = 'centered_age'
cats = np.array([1947, 1963, 1975, 1985, 1995])
target_key = 'gender'
cats = np.array(['male', 'female']) # male is dropped -> female mapped to 1
return cats, target_key
def save_sparse(matrix:sparse.csr_matrix, path:str):
sparse.save_npz(path, matrix)
print('saved sparse matrix to file...')
def load_sparse(path:str):
matrix = sparse.load_npz(path+'.npz')
print('loaded sparse matrix from file...')
return matrix
def save_uids(uids, path:str):
with open(path, 'wb') as w:
pickle.dump(uids, w)
def load_uids(path:str):
with open(path, 'rb') as w:
uids= pickle.load(w)
return uids
This diff is collapsed.
......@@ -377,7 +377,7 @@ if __name__ == "__main__":
args["path"] = "../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed"
args["labelpath"] = "../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/labels.ndjson"
args["workset"] ='workset'
args["part"] =['creator']
args["part"] =['creator', 'performer']
args["minchars"] = [100, 250, 500]
args["authors"] = [1000, 500, 150, 50]
args["threshold"] = 0.1
#!/usr/bin/env python3
import pandas as pd
import numpy as np
import ndjson
import jsonlines
import json
import os
import sys
import random as rd
import json
import multiprocessing as mp
import re, regex
import gc
import psutil
import time
import psutil
datapath = '/cobra/ptmp/mschuber/PAN/Data/pan19-celebrity-profiling-training-dataset-2019-01-31/'
#datapath = '../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/'
combinefile = 'combined_preprocess'
combined = 'combined_mult.ndjson'
#regex for rpeprocessing
http = re.compile(r'(https{0,1}:\S+)')
www = re.compile(r'(www\.\S+)')
emoji = regex.compile(r'\X')
tagging = re.compile(r'#\S+')
retweet = re.compile(r'(RT\s{0,1}@\w+(?![@,\\])[\w]:{0,1})')
mention = re.compile(r'(@\w+(?![@,\\])\w)')
def chunkify(fname,size=np.NaN):
linebytes = []
if np.isnan(size):
print('size is nan...look at single lines only')
i = 0
#fileEnd = os.path.getsize(fname)
with open(fname, 'r', encoding='utf-8') as f:
nextLineByte = f.tell()
while True:
line = f.readline()
nextLineByte = f.tell() #returns the location of the next line
#if i == 10000:
# break
if not line or line == '':
sys.exit('Chunking not yet implemented')
return linebytes
def preprocess(dic):
#tweet_pre = ['§BEGIN§']
tweet_pre = []
tweet_pre.extend(retweet.split(dic.pop('text_org', None)))
tweet_pre = [retweet.sub('§RETWEET§', tweet_pre[i]) for i in range(0, len(tweet_pre))]
tmp_tweet = []
#find and substitute HTTP-URLs
for i in range(0, len(tweet_pre)):
tmp = http.split(tweet_pre[i])
tmp = [http.sub('§LINK§', tmp[j]) for j in range(0, len(tmp)) if tmp[j] != '']
tweet_pre = tmp_tweet
tmp_tweet = []
#find and substitute www-URLs
for i in range(0, len(tweet_pre)):
tmp = www.split(tweet_pre[i])
tmp = [www.sub('§LINK§', tmp[j]) for j in range(0, len(tmp)) if tmp[j] != '']
tweet_pre = tmp_tweet
tmp_tweet = []
#find and substitute mentions
for i in range(0, len(tweet_pre)):
tmp = mention.split(tweet_pre[i])
tmp = [mention.sub('§MENTION§', tmp[j]) for j in range(0, len(tmp)) if tmp[j] != '']
tweet_pre = tmp_tweet
tmp_tweet = []
#find and substitute Tagging
for i in range(0, len(tweet_pre)):
tmp = tagging.split(tweet_pre[i])
tmp = [tagging.sub('§TAG§', tmp[j]) for j in range(0, len(tmp)) if tmp[j] != '']
tweet_pre = tmp_tweet
tmp_tweet = []
#make character lists; keep unicode emojis as well as subsitutes as singel individual character
for i in range(0, len(tweet_pre)):
if tweet_pre[i] not in ['§RETWEET§', '§LINK§', '§MENTION§', '§TAGGING§', '§BEGIN§', '§END§']:
tmp = emoji.findall(tweet_pre[i])
#dic['text_pre'] = tmp_tweet
#del tmp_tweet
return dic, tmp_tweet
def ngrams(charlist):
bigrams = []
trigrams = []
quadgrams = []
leng = len(charlist)
for i in range(0, leng):
if i <leng-4:
quadgrams.append(charlist[i] + charlist[i+1] +
charlist[i+2] + charlist[i+3])
trigrams.append(charlist[i] + charlist[i+1] + charlist[i+2])
bigrams.append(charlist[i] + charlist[i+1])
elif i <leng-3:
trigrams.append(charlist[i] + charlist[i+1] + charlist[i+2])
bigrams.append(charlist[i] + charlist[i+1])
elif i<leng-1:
bigrams.append(charlist[i] + charlist[i+1])
return [bigrams, trigrams, quadgrams]
def process_wrapper(chunkStart, q):
res = []
grams = []
with open(datapath+combined, 'r', encoding='utf-8') as f:
lines = f.readline()
lines = ndjson.loads(lines)[0]
dic, text_pre = preprocess(lines)
del text_pre
del dic
tmp = ngrams(grams[0])
del lines
del grams
del tmp
return 'finished'
def listener(q):
'''listens for messages on the q, writes to file. '''
#print('finished making files..')
prof = ['sports', 'performer', 'creator', 'politics', 'manager','science', 'professional', 'religious']
writers = []
filehandles = []
pathmodify = ['singlegrams/', 'bigrams/', 'trigrams/', 'quadgrams/']
types =['', '_bigrams', '_trigrams', '_quadgrams']
for i in range(0, len(types)):
item = types[i]
gramwriter = []
f = open(datapath+pathmodify[i]+combinefile+'{}.ndjson'.format(item), mode='a', encoding='utf-8')
writer = jsonlines.Writer(f, flush=True)
for j in range(0, len(prof)):
job = prof[j]
f_p = open(datapath+pathmodify[i]+combinefile+'{}_{}.ndjson'.format(item, job), mode='a', encoding='utf-8')
writer_p = jsonlines.Writer(f_p, flush=True)
print('made writers...')
while True:
print('waiting for item from queue...')
m = q.get() ##Q:
print('got item from queue')
if m == 'kill':
dic = m[0]
grams = m[1:5]
index = prof.index(dic['occupation'])+1
del m
for i in range(0, len(types)):
dic['text_pre'] = grams[0][i]
del dic
del grams
for r in range(0, len(writers)):
for w in range(0, len(writers[r])):
for r in range(0,len(filehandles)):
def make_file_paths():
prof = ['sports', 'performer', 'creator', 'politics', 'manager','science', 'professional', 'religious']
pathmodify = ['singlegrams/', 'bigrams/', 'trigrams/', 'quadgrams/']
types =['', '_bigrams', '_trigrams', '_quadgrams']
for el in pathmodify:
os.makedirs(datapath+el, exist_ok=True)
for i in range(0, len(types)):
item = types[i]
f = open(datapath+pathmodify[i]+combinefile+'{}.ndjson'.format(item), mode='w', encoding='utf-8')
for j in range(0, len(prof)):