bss/dr.botzo
1
0
Fork 0
Code Issues 3 Pull Requests Releases 2 Wiki Activity

reorganizing project directories, part 1

Browse Source
This commit is contained in:
Brian S. Stephan
2015-05-08 20:37:22 -05:00
parent 827131eeda
commit 2ffdf26ec2
107 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
markov/__init__.py
View File

9
markov/admin.py
View File

@@ -1,9 +0,0 @@
from django.contrib import admin
from markov.models import MarkovContext, MarkovTarget, MarkovState
admin.site.register(MarkovContext)
admin.site.register(MarkovTarget)
admin.site.register(MarkovState)
# vi:tabstop=4:expandtab:autoindent

36
markov/forms.py
View File

@@ -1,36 +0,0 @@
"""
markov/forms.py --- forms for manipulating markov data
"""
import logging
from django.forms import Form, CharField, FileField, ModelChoiceField
from markov.models import MarkovContext
log = logging.getLogger('dr_botzo.markov')
class LogUploadForm(Form):
"""Accept a file upload that will be imported into Markov stuff."""
log_file = FileField(help_text="Weechat log format.")
context = ModelChoiceField(queryset=MarkovContext.objects.all())
ignore_nicks = CharField(help_text="Comma-separated list of nicks to ignore.",
required=False)
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip.",
required=False)
class TeachLineForm(Form):
"""Accept a line that will be imported into Markov stuff."""
context = ModelChoiceField(queryset=MarkovContext.objects.all())
line = CharField()
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip.",
required=False)
# vi:tabstop=4:expandtab:autoindent

80
markov/migrations/0001_markov_context_and_target_and_state.py
View File

@@ -1,80 +0,0 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'MarkovContext'
db.create_table(u'markov_markovcontext', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=32)),
))
db.send_create_signal(u'markov', ['MarkovContext'])
# Adding model 'MarkovTarget'
db.create_table(u'markov_markovtarget', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('target', self.gf('django.db.models.fields.CharField')(max_length=64)),
('context', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['markov.MarkovContext'])),
('chatter_chance', self.gf('django.db.models.fields.IntegerField')(default=0)),
))
db.send_create_signal(u'markov', ['MarkovTarget'])
# Adding model 'MarkovState'
db.create_table(u'markov_markovstate', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('k1', self.gf('django.db.models.fields.CharField')(max_length=128)),
('k2', self.gf('django.db.models.fields.CharField')(max_length=128)),
('v', self.gf('django.db.models.fields.CharField')(max_length=128)),
('count', self.gf('django.db.models.fields.IntegerField')(default=0)),
('context', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['markov.MarkovContext'])),
))
db.send_create_signal(u'markov', ['MarkovState'])
# Adding unique constraint on 'MarkovState', fields ['context', 'k1', 'k2', 'v']
db.create_unique(u'markov_markovstate', ['context_id', 'k1', 'k2', 'v'])
def backwards(self, orm):
# Removing unique constraint on 'MarkovState', fields ['context', 'k1', 'k2', 'v']
db.delete_unique(u'markov_markovstate', ['context_id', 'k1', 'k2', 'v'])
# Deleting model 'MarkovContext'
db.delete_table(u'markov_markovcontext')
# Deleting model 'MarkovTarget'
db.delete_table(u'markov_markovtarget')
# Deleting model 'MarkovState'
db.delete_table(u'markov_markovstate')
models = {
u'markov.markovcontext': {
'Meta': {'object_name': 'MarkovContext'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
},
u'markov.markovstate': {
'Meta': {'unique_together': "(('context', 'k1', 'k2', 'v'),)", 'object_name': 'MarkovState'},
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
'count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'k1': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'k2': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'v': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
u'markov.markovtarget': {
'Meta': {'object_name': 'MarkovTarget'},
'chatter_chance': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'target': ('django.db.models.fields.CharField', [], {'max_length': '64'})
}
}
complete_apps = ['markov']

60
markov/migrations/0002_unique_target_name_and_context_name.py
View File

@@ -1,60 +0,0 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting field 'MarkovTarget.target'
db.delete_column(u'markov_markovtarget', 'target')
# Adding field 'MarkovTarget.name'
db.add_column(u'markov_markovtarget', 'name',
self.gf('django.db.models.fields.CharField')(default='', unique=True, max_length=64),
keep_default=False)
# Adding unique constraint on 'MarkovContext', fields ['name']
db.create_unique(u'markov_markovcontext', ['name'])
def backwards(self, orm):
# Removing unique constraint on 'MarkovContext', fields ['name']
db.delete_unique(u'markov_markovcontext', ['name'])
# Adding field 'MarkovTarget.target'
db.add_column(u'markov_markovtarget', 'target',
self.gf('django.db.models.fields.CharField')(default='', max_length=64),
keep_default=False)
# Deleting field 'MarkovTarget.name'
db.delete_column(u'markov_markovtarget', 'name')
models = {
u'markov.markovcontext': {
'Meta': {'object_name': 'MarkovContext'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
},
u'markov.markovstate': {
'Meta': {'unique_together': "(('context', 'k1', 'k2', 'v'),)", 'object_name': 'MarkovState'},
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
'count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'k1': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'k2': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'v': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
u'markov.markovtarget': {
'Meta': {'object_name': 'MarkovTarget'},
'chatter_chance': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
}
}
complete_apps = ['markov']

44
markov/migrations/0003_markov_context_max_length_64.py
View File

@@ -1,44 +0,0 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Changing field 'MarkovContext.name'
db.alter_column(u'markov_markovcontext', 'name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=64))
def backwards(self, orm):
# Changing field 'MarkovContext.name'
db.alter_column(u'markov_markovcontext', 'name', self.gf('django.db.models.fields.CharField')(max_length=32, unique=True))
models = {
u'markov.markovcontext': {
'Meta': {'object_name': 'MarkovContext'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
},
u'markov.markovstate': {
'Meta': {'unique_together': "(('context', 'k1', 'k2', 'v'),)", 'object_name': 'MarkovState'},
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
'count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'k1': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'k2': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'v': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
u'markov.markovtarget': {
'Meta': {'object_name': 'MarkovTarget'},
'chatter_chance': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
}
}
complete_apps = ['markov']

50
markov/migrations/0004_more_indexes_on_markov_state.py
View File

@@ -1,50 +0,0 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding index on 'MarkovState', fields ['context', 'k1', 'k2']
db.create_index(u'markov_markovstate', ['context_id', 'k1', 'k2'])
# Adding index on 'MarkovState', fields ['context', 'v']
db.create_index(u'markov_markovstate', ['context_id', 'v'])
def backwards(self, orm):
# Removing index on 'MarkovState', fields ['context', 'v']
db.delete_index(u'markov_markovstate', ['context_id', 'v'])
# Removing index on 'MarkovState', fields ['context', 'k1', 'k2']
db.delete_index(u'markov_markovstate', ['context_id', 'k1', 'k2'])
models = {
u'markov.markovcontext': {
'Meta': {'object_name': 'MarkovContext'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
},
u'markov.markovstate': {
'Meta': {'unique_together': "(('context', 'k1', 'k2', 'v'),)", 'object_name': 'MarkovState', 'index_together': "[['context', 'k1', 'k2'], ['context', 'v']]"},
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
'count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'k1': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'k2': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'v': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
u'markov.markovtarget': {
'Meta': {'object_name': 'MarkovTarget'},
'chatter_chance': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'context': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['markov.MarkovContext']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
}
}
complete_apps = ['markov']

0
markov/migrations/__init__.py
View File

72
markov/models.py
View File

@@ -1,72 +0,0 @@
"""
markov/models.py --- save brain pieces for chaining
"""
import logging
from django.db import models
log = logging.getLogger('dr_botzo.markov')
class MarkovContext(models.Model):
"""Define contexts for Markov chains."""
name = models.CharField(max_length=64, unique=True)
def __unicode__(self):
"""String representation."""
return u"{0:s}".format(self.name)
class MarkovTarget(models.Model):
"""Define IRC targets that relate to a context, and can occasionally be talked to."""
name = models.CharField(max_length=64, unique=True)
context = models.ForeignKey(MarkovContext)
chatter_chance = models.IntegerField(default=0)
def __unicode__(self):
"""String representation."""
return u"{0:s}".format(self.name)
class MarkovState(models.Model):
"""One element in a Markov chain, some text or something."""
_start1 = '__start1'
_start2 = '__start2'
_stop = '__stop'
k1 = models.CharField(max_length=128)
k2 = models.CharField(max_length=128)
v = models.CharField(max_length=128)
count = models.IntegerField(default=0)
context = models.ForeignKey(MarkovContext)
class Meta:
index_together = [
['context', 'k1', 'k2'],
['context', 'v'],
]
permissions = {
('import_log_file', "Can import states from a log file"),
('teach_line', "Can teach lines"),
}
unique_together = ('context', 'k1', 'k2', 'v')
def __unicode__(self):
"""String representation."""
return u"{0:s},{1:s} -> {2:s} (count: {3:d})".format(self.k1, self.k2, self.v, self.count)
# vi:tabstop=4:expandtab:autoindent

15
markov/urls.py
View File

@@ -1,15 +0,0 @@
"""
markov/urls.py --- url patterns for markov stuff
"""
from django.conf.urls import patterns, url
urlpatterns = patterns('markov.views',
url(r'^$', 'index', name='markov_index'),
url(r'^context/(?P<context_id>\d+)/$', 'context_index', name='markov_context_index'),
url(r'^import/$', 'import_file', name='markov_import_file'),
url(r'^teach/$', 'teach_line', name='markov_teach_line'),
)
# vi:tabstop=4:expandtab:autoindent

223
markov/views.py
View File

@@ -1,223 +0,0 @@
"""
markov/views.py --- manipulate markov data
"""
import logging
import random
import time
from django.contrib.auth.decorators import permission_required
from django.db.models import Sum
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render
from markov.forms import LogUploadForm, TeachLineForm
from markov.models import MarkovContext, MarkovTarget, MarkovState
log = logging.getLogger('dr_botzo.markov')
def index(request):
"""Display nothing, for the moment."""
return HttpResponse()
def context_index(request, context_id):
"""Display the context index for the given context."""
start_t = time.time()
context = get_object_or_404(MarkovContext, pk=context_id)
chain = u" ".join(_generate_line(context))
end_t = time.time()
return render(request, 'markov/context.html', {'chain': chain,
'context': context,
'elapsed': end_t - start_t})
@permission_required('import_log_file', raise_exception=True)
def import_file(request):
"""Accept a file upload and turn it into markov stuff.
Current file formats supported:
* weechat
"""
if request.method == 'POST':
form = LogUploadForm(request.POST, request.FILES)
if form.is_valid():
log_file = request.FILES['log_file']
context = form.cleaned_data['context']
ignores = form.cleaned_data['ignore_nicks'].split(',')
strips = form.cleaned_data['strip_prefixes'].split(' ')
whos = []
for line in log_file:
(timestamp, who, what) = line.decode('utf-8').split('\t', 2)
if who in ('-->', '<--', '--', ' *'):
continue
if who in ignores:
continue
whos.append(who)
# this is a line we probably care about now
what = [x for x in what.rstrip().split(' ') if x not in strips]
_learn_line(' '.join(what), context)
log.debug(set(whos))
else:
form = LogUploadForm()
return render(request, 'markov/import_file.html', {'form': form})
@permission_required('teach_line', raise_exception=True)
def teach_line(request):
"""Teach one line directly."""
if request.method == 'POST':
form = TeachLineForm(request.POST)
if form.is_valid():
line = form.cleaned_data['line']
context = form.cleaned_data['context']
strips = form.cleaned_data['strip_prefixes'].split(' ')
what = [x for x in line.rstrip().split(' ') if x not in strips]
_learn_line(' '.join(what), context)
else:
form = TeachLineForm()
return render(request, 'markov/teach_line.html', {'form': form})
def _generate_sentence(context, topics=None, max_words=30):
"""Generate a Markov chain."""
words = []
# if we have topics, try to work from it and work backwards
if topics:
topic_word = random.choice(topics)
topics.remove(topic_word)
log.debug(u"looking for topic '{0:s}'".format(topic_word))
new_states = MarkovState.objects.filter(context=context, v=topic_word)
if len(new_states) > 0:
log.debug(u"found '{0:s}', starting backwards".format(topic_word))
words.insert(0, topic_word)
while len(words) <= max_words and words[0] != MarkovState._start2:
log.debug(u"looking backwards for '{0:s}'".format(words[0]))
new_states = MarkovState.objects.filter(context=context, v=words[0])
words.insert(0, _get_word_out_of_states(new_states, backwards=True))
# if we didn't get topic stuff, we need to start (forwards) here
if len(words) == 0:
words = [MarkovState._start1, MarkovState._start2]
i = len(words)
while len(words) <= max_words and words[-1] != MarkovState._stop:
log.debug(u"looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
log.debug(u"states retrieved")
words.append(_get_word_out_of_states(new_states))
i += 1
words = [word for word in words if word not in
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
return words
def _generate_longish_sentence(context, topics=None, min_words=4, max_words=30):
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
tries = 0
while tries < 5:
sent = _generate_sentence(context, topics=topics, max_words=max_words)
if len(sent) >= min_words:
return sent
tries += 1
# if we got here, we need to just give up
return _generate_sentence(context)
def _generate_line(context, topics=None, min_words=15, max_words=30, max_sentences=3):
"""String multiple sentences together into a coherent sentence."""
tries = 0
sentences = 0
line = []
while tries < 5:
line += _generate_longish_sentence(context, topics=topics, max_words=max_words)
sentences += 1
if sentences >= max_sentences:
return line
if len(line) >= min_words:
return line
else:
if line[-1][-1] not in [',', '.', '!']:
line[-1] += random.choice([',', '.', '!'])
tries += 1
# if we got here, we need to give up
return line
def _get_word_out_of_states(states, backwards=False):
"""Pick one random word out of the given states."""
new_word = ''
running = 0
count_sum = states.aggregate(Sum('count'))['count__sum']
hit = random.randint(0, count_sum)
log.debug(u"sum: {0:d} hit: {1:d}".format(count_sum, hit))
states_itr = states.iterator()
for state in states_itr:
running += state.count
if running >= hit:
if backwards:
new_word = state.k2
else:
new_word = state.v
break
log.debug(u"found '{0:s}'".format(new_word))
return new_word
def _learn_line(line, context):
"""Create a bunch of MarkovStates for a given line of text."""
log.debug(u"learning {0:.40s}...".format(line))
words = line.split()
words = [MarkovState._start1, MarkovState._start2] + words + [MarkovState._stop]
for word in words:
if len(word) > MarkovState._meta.get_field('k1').max_length:
return
for i, word in enumerate(words):
log.debug(u"'{0:s}','{1:s}' -> '{2:s}'".format(words[i], words[i+1], words[i+2]))
state, created = MarkovState.objects.get_or_create(context=context,
k1=words[i],
k2=words[i+1],
v=words[i+2])
state.count += 1
state.save()
if i > len(words) - 4:
break
# vi:tabstop=4:expandtab:autoindent