Path Lengths¶
In [1]:
import pandas as pd
import numpy as np
import json
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
path = "/Users/mark/Dropbox/Math/Complex_Systems/research/wikipedia-network/paper/writeup/graphics/"
In [2]:
#load results into dataframe (~ 1min runtime)
results_path = "/Users/mark/Desktop/wiki_v4/"
with open(results_path + "lengths.json") as f:
dict = json.load(f)
df = pd.DataFrame(dict.items())
df.columns = ['page', 'path length']
Articles with the Longest Path Length¶
path length: number of first link traversals up to a repeated article or an invalid link
In [6]:
df.sort(columns='path length', ascending=False).head(50)
Out[6]:
Excluding Liturgics...¶
In [3]:
top1k= df.sort(columns='path length', ascending=False).head(1000)
top1k[top1k['page'].apply(lambda e: "liturgics" not in e.lower())]
Out[3]:
What's the total path length traversed?¶
In [52]:
df['path length'].sum()
Out[52]:
How many pages lead to dead links (or themselves)?¶
In [20]:
df[df['path length'] == 0].count()
Out[20]:
a sample:¶
In [21]:
df[df['path length'] == 0].head(10)
Out[21]:
How many articles have a path length of 2?¶
includes invalid links
In [48]:
df[df['path length'] == 2].count()
Out[48]:
How many articles have a path length of 3?¶
includes invalid links
In [50]:
df[df['path length'] == 3].count()
Out[50]:
What's the most frequent path length?¶
In [45]:
df['path length'].mode()
Out[45]:
How many pages have a path length of 29?¶
In [46]:
df[df['path length'] == 29].count()
Out[46]:
Summary Statistics¶
In [25]:
df.describe()
Out[25]:
75% of pages traverse fewer than 30 first links!
In [8]:
plt.figure(figsize=(8,6))
sns.set_style("whitegrid")
sns.boxplot(x=df["path length"])
plt.savefig(path+'path_lengths_boxplot.png', format='png', dpi=300, bbox_inches='tight')
Density Plots¶
In [4]:
plt.figure(figsize=(8,6))
sns.distplot(df["path length"])
Out[4]:
In [5]:
# runtime ~3min
plt.figure(figsize=(8,6))
with sns.axes_style("whitegrid"):
sns.kdeplot(df["path length"], shade=True, legend=False, color='g')
sns.despine(left=True)
#make axis font size larger
plt.tick_params(axis='both', which='major', labelsize=14)
#labels
plt.xlabel("path length", fontsize=14)
plt.ylabel("density", fontsize=14)
in log space¶
In [6]:
# runtime ~3min
plt.figure(figsize=(8,6))
with sns.axes_style("whitegrid"):
sns.kdeplot(np.log10(df["path length"] + 1), shade=True, legend=False, color='g')
sns.despine(left=True)
#make axis font size larger
plt.tick_params(axis='both', which='major', labelsize=14)
#labels
plt.xlabel("$\log_{10}$(path length)", fontsize=14)
plt.ylabel("density", fontsize=14)
combined plot¶
In [ ]:
#defaults
sns.set()
plt.figure(figsize=(8,6))
with sns.axes_style("whitegrid"):
sns.kdeplot(df["path length"], shade=True, legend=False, color='g')
sns.despine(left=True)
#make axis font size larger
plt.tick_params(axis='both', which='major', labelsize=14)
#labels
plt.xlabel("path length", fontsize=14)
plt.ylabel("density", fontsize=14)
#define plot axis limits
axes = plt.gca()
xticks = axes.xaxis.get_major_ticks()
xticks[0].label1.set_visible(False)
yticks = axes.yaxis.get_major_ticks()
yticks[0].label1.set_visible(False)
sns.set_style("dark")
#subplot in top corner
a = plt.axes([.53, .51, .37, .37], axisbg='y')
with sns.axes_style("whitegrid"):
sns.kdeplot(np.log10(df["path length"] + 1), shade=True, legend=False, color='g')
sns.despine(left=True)
#make axis font size larger
plt.tick_params(axis='both', which='major', labelsize=14)
#labels
plt.xlabel("$\log_{10}$(path length)", fontsize=14)
plt.ylabel("density", fontsize=14)
#define plot axis limits
axes = plt.gca()
xticks = axes.xaxis.get_major_ticks()
xticks[0].label1.set_visible(False)
yticks = axes.yaxis.get_major_ticks()
yticks[0].label1.set_visible(False)
#transparent
a.patch.set_alpha(0.1)
#back to defaults
sns.set()
#save figure
plt.savefig(path+'path_lengths_dist.png', format='png', dpi=300, bbox_inches='tight')
Distribution of Path Lengths¶
In [41]:
toppdf_10k = df.sort(columns='path length', ascending=False).head(10000)['path length']
toppdf_1k = df.sort(columns='path length', ascending=False).head(1000)['path length']
toppdf_100 = df.sort(columns='path length', ascending=False).head(100)['path length']
In [31]:
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
In [40]:
visits_plot1 = toppdf_10k.plot(kind='hist', title="Pages with Longest Paths (top 10k)")
visits_plot1.set_xlabel("path length")
visits_plot1.set_ylabel("frequency")
Out[40]:
In [42]:
visits_plot2 = toppdf_1k.plot(kind='hist', title="Pages with Longest Paths (top 1k)")
visits_plot2.set_xlabel("path length")
visits_plot2.set_ylabel("frequency")
Out[42]:
In [44]:
visits_plot3 = toppdf_100.plot(kind='hist', title="Pages with Longest Paths (top 100)")
visits_plot3.set_xlabel("path length")
visits_plot3.set_ylabel("frequency")
Out[44]:
