{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### The Time Machine (loaded)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# tokens: 13 ['the', 'time', 'traveller', 'for', 'so', 'it', 'will', 'be', 'convenient', 'to', 'speak', 'of', 'him']\n",
"# tokens: 12 ['was', 'expounding', 'a', 'recondite', 'matter', 'to', 'us', 'his', 'grey', 'eyes', 'shone', 'and']\n"
]
}
],
"source": [
"import sys\n",
"sys.path.insert(0, '..')\n",
"\n",
"import collections\n",
"import re\n",
"with open('../data/timemachine.txt', 'r') as f:\n",
" lines = f.readlines()\n",
" raw_dataset = [re.sub('[^A-Za-z]+', ' ', st).lower().split() for st in lines]\n",
"\n",
"# Let's read the first 10 lines of the text \n",
"for st in raw_dataset[8:10]:\n",
" print('# tokens:', len(st), st) "
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Word Counts"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"frequency of 'traveller': 61\n",
"[('the', 2261), ('i', 1267), ('and', 1245), ('of', 1155), ('a', 816), ('to', 695), ('was', 552), ('in', 541), ('that', 443), ('my', 440)]\n"
]
}
],
"source": [
"counter = collections.Counter([tk for st in raw_dataset for tk in st])\n",
"print(\"frequency of 'traveller':\", counter['traveller'])\n",
"# Print the 10 most frequent words with word frequency count\n",
"print(counter.most_common(10))"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Frequency Statistics"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/var/folders/wx/wk4cq72n2t5943_ddkmw50s40000gn/T/ipykernel_54523/1351866829.py:4: DeprecationWarning: `set_matplotlib_formats` is deprecated since IPython 7.23, directly use `matplotlib_inline.backend_inline.set_matplotlib_formats()`\n",
" display.set_matplotlib_formats('svg')\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%matplotlib inline\n",
"from matplotlib import pyplot as plt\n",
"from IPython import display\n",
"display.set_matplotlib_formats('svg')\n",
"\n",
"wordcounts = [count for _,count in counter.most_common()]\n",
"plt.loglog(wordcounts);"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Zipf's Law\n",
"\n",
"$$n(x) \\propto (x + c)^{-\\alpha} \\text{ and hence }\n",
"\\log n(x) = -\\alpha \\log (x+c) + \\mathrm{const.}$$\n",
"\n",
"Does it work for word pairs, too?"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Beginning of the book\n",
" [('the', 'time'), ('time', 'machine'), ('machine', 'by'), ('by', 'h'), ('h', 'g'), ('g', 'wells'), ('wells', 'i'), ('i', 'the'), ('the', 'time'), ('time', 'traveller')]\n",
"Most common word pairs\n",
" [(('of', 'the'), 309), (('in', 'the'), 169), (('i', 'had'), 130), (('i', 'was'), 112), (('and', 'the'), 109), (('the', 'time'), 102), (('it', 'was'), 99), (('to', 'the'), 85), (('as', 'i'), 78), (('of', 'a'), 73)]\n"
]
}
],
"source": [
"wseq = [tk for st in raw_dataset for tk in st]\n",
"word_pairs = [pair for pair in zip(wseq[:-1], wseq[1:])]\n",
"print('Beginning of the book\\n', word_pairs[:10])\n",
"counter_pairs = collections.Counter(word_pairs)\n",
"print('Most common word pairs\\n', counter_pairs.most_common(10))"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Frequency Statistics"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"word_triples = [triple for triple in zip(wseq[:-2], wseq[1:-1], wseq[2:])]\n",
"counter_triples = collections.Counter(word_triples)\n",
"\n",
"bigramcounts = [count for _,count in counter_pairs.most_common()]\n",
"triplecounts = [count for _,count in counter_triples.most_common()]\n",
"plt.loglog(wordcounts, label='word counts');\n",
"plt.loglog(bigramcounts, label='bigram counts');\n",
"plt.loglog(triplecounts, label='triple counts');\n",
"plt.legend();"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"celltoolbar": "Slideshow",
"kernelspec": {
"display_name": "datasci415",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}