script to take a file of lines and create sqlite markov chains
This commit is contained in:
parent
1712a7db53
commit
3d6ede5155
|
@ -0,0 +1,53 @@
|
||||||
|
"""
|
||||||
|
import-file-into-markov_chain.py
|
||||||
|
Copyright (C) 2011 Brian S. Stephan
|
||||||
|
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import fileinput
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
db = sqlite3.connect('dr.botzo.data')
|
||||||
|
db.row_factory = sqlite3.Row
|
||||||
|
|
||||||
|
cur = db.cursor()
|
||||||
|
statement = 'INSERT INTO markov_chain (k1, k2, v) VALUES (?, ?, ?)'
|
||||||
|
for line in fileinput.input():
|
||||||
|
# set up the head of the chain
|
||||||
|
w1 = '__start1'
|
||||||
|
w2 = '__start2'
|
||||||
|
|
||||||
|
# for each word pair, add the next word to the dictionary
|
||||||
|
for word in line.split():
|
||||||
|
try:
|
||||||
|
cur.execute(statement, (w1.decode('utf-8', 'replace').lower(), w2.decode('utf-8', 'replace').lower(), word.decode('utf-8', 'replace').lower()))
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
db.rollback()
|
||||||
|
print("sqlite error: " + str(e))
|
||||||
|
raise
|
||||||
|
|
||||||
|
w1, w2 = w2, word
|
||||||
|
|
||||||
|
try:
|
||||||
|
cur.execute(statement, (w1.decode('utf-8', 'replace').lower(), w2.decode('utf-8', 'replace').lower(), '__stop'))
|
||||||
|
db.commit()
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
db.rollback()
|
||||||
|
print("sqlite error: " + str(e))
|
||||||
|
raise
|
||||||
|
|
||||||
|
# vi:tabstop=4:expandtab:autoindent
|
Loading…
Reference in New Issue