markov: move shared methods into lib.py
This commit is contained in:
parent
1570bc28f9
commit
c371df5eb0
|
@ -0,0 +1,150 @@
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
|
||||||
|
from django.db.models import Sum
|
||||||
|
|
||||||
|
from markov.models import MarkovContext, MarkovState, MarkovTarget
|
||||||
|
|
||||||
|
|
||||||
|
log = logging.getLogger('markov.lib')
|
||||||
|
|
||||||
|
|
||||||
|
def generate_line(context, topics=None, min_words=15, max_words=30, max_sentences=3):
|
||||||
|
"""String multiple sentences together into a coherent sentence."""
|
||||||
|
|
||||||
|
tries = 0
|
||||||
|
sentences = 0
|
||||||
|
line = []
|
||||||
|
while tries < 5:
|
||||||
|
line += generate_longish_sentence(context, topics=topics, max_words=max_words)
|
||||||
|
sentences += 1
|
||||||
|
if sentences >= max_sentences:
|
||||||
|
return line
|
||||||
|
if len(line) >= min_words:
|
||||||
|
return line
|
||||||
|
else:
|
||||||
|
if line[-1][-1] not in [',', '.', '!']:
|
||||||
|
line[-1] += random.choice([',', '.', '!'])
|
||||||
|
|
||||||
|
tries += 1
|
||||||
|
|
||||||
|
# if we got here, we need to give up
|
||||||
|
return line
|
||||||
|
|
||||||
|
|
||||||
|
def generate_longish_sentence(context, topics=None, min_words=4, max_words=30):
|
||||||
|
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
|
||||||
|
|
||||||
|
tries = 0
|
||||||
|
while tries < 5:
|
||||||
|
sent = generate_sentence(context, topics=topics, max_words=max_words)
|
||||||
|
if len(sent) >= min_words:
|
||||||
|
return sent
|
||||||
|
|
||||||
|
tries += 1
|
||||||
|
|
||||||
|
# if we got here, we need to just give up
|
||||||
|
return generate_sentence(context)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_sentence(context, topics=None, max_words=30):
|
||||||
|
"""Generate a Markov chain."""
|
||||||
|
|
||||||
|
words = []
|
||||||
|
# if we have topics, try to work from it and work backwards
|
||||||
|
if topics:
|
||||||
|
topic_word = random.choice(topics)
|
||||||
|
topics.remove(topic_word)
|
||||||
|
log.debug(u"looking for topic '{0:s}'".format(topic_word))
|
||||||
|
new_states = MarkovState.objects.filter(context=context, v=topic_word)
|
||||||
|
|
||||||
|
if len(new_states) > 0:
|
||||||
|
log.debug(u"found '{0:s}', starting backwards".format(topic_word))
|
||||||
|
words.insert(0, topic_word)
|
||||||
|
while len(words) <= max_words and words[0] != MarkovState._start2:
|
||||||
|
log.debug(u"looking backwards for '{0:s}'".format(words[0]))
|
||||||
|
new_states = MarkovState.objects.filter(context=context, v=words[0])
|
||||||
|
words.insert(0, get_word_out_of_states(new_states, backwards=True))
|
||||||
|
|
||||||
|
# if we didn't get topic stuff, we need to start (forwards) here
|
||||||
|
if len(words) == 0:
|
||||||
|
words = [MarkovState._start1, MarkovState._start2]
|
||||||
|
|
||||||
|
i = len(words)
|
||||||
|
while len(words) <= max_words and words[-1] != MarkovState._stop:
|
||||||
|
log.debug(u"looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
|
||||||
|
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
|
||||||
|
log.debug(u"states retrieved")
|
||||||
|
words.append(get_word_out_of_states(new_states))
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
words = [word for word in words if word not in
|
||||||
|
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
|
||||||
|
|
||||||
|
return words
|
||||||
|
|
||||||
|
|
||||||
|
def get_or_create_target_context(target_name):
|
||||||
|
"""Return the context for a provided nick/channel, creating missing ones."""
|
||||||
|
|
||||||
|
# find the stuff, or create it
|
||||||
|
target, c = MarkovTarget.objects.get_or_create(name=target_name)
|
||||||
|
try:
|
||||||
|
return target.context
|
||||||
|
except MarkovContext.DoesNotExist:
|
||||||
|
# make a context --- lacking a good idea, just create one with this target name until configured otherwise
|
||||||
|
context, c = MarkovContext.objects.get_or_create(name=target_name)
|
||||||
|
target.context = context
|
||||||
|
target.save()
|
||||||
|
|
||||||
|
return target.context
|
||||||
|
|
||||||
|
|
||||||
|
def get_word_out_of_states(states, backwards=False):
|
||||||
|
"""Pick one random word out of the given states."""
|
||||||
|
|
||||||
|
new_word = ''
|
||||||
|
running = 0
|
||||||
|
count_sum = states.aggregate(Sum('count'))['count__sum']
|
||||||
|
hit = random.randint(0, count_sum)
|
||||||
|
|
||||||
|
log.debug(u"sum: {0:d} hit: {1:d}".format(count_sum, hit))
|
||||||
|
|
||||||
|
states_itr = states.iterator()
|
||||||
|
for state in states_itr:
|
||||||
|
running += state.count
|
||||||
|
if running >= hit:
|
||||||
|
if backwards:
|
||||||
|
new_word = state.k2
|
||||||
|
else:
|
||||||
|
new_word = state.v
|
||||||
|
|
||||||
|
break
|
||||||
|
|
||||||
|
log.debug(u"found '{0:s}'".format(new_word))
|
||||||
|
return new_word
|
||||||
|
|
||||||
|
|
||||||
|
def learn_line(line, context):
|
||||||
|
"""Create a bunch of MarkovStates for a given line of text."""
|
||||||
|
|
||||||
|
log.debug(u"learning %s...", line[:40])
|
||||||
|
|
||||||
|
words = line.split()
|
||||||
|
words = [MarkovState._start1, MarkovState._start2] + words + [MarkovState._stop]
|
||||||
|
|
||||||
|
for word in words:
|
||||||
|
if len(word) > MarkovState._meta.get_field('k1').max_length:
|
||||||
|
return
|
||||||
|
|
||||||
|
for i, word in enumerate(words):
|
||||||
|
log.debug(u"'{0:s}','{1:s}' -> '{2:s}'".format(words[i], words[i+1], words[i+2]))
|
||||||
|
state, created = MarkovState.objects.get_or_create(context=context,
|
||||||
|
k1=words[i],
|
||||||
|
k2=words[i+1],
|
||||||
|
v=words[i+2])
|
||||||
|
state.count += 1
|
||||||
|
state.save()
|
||||||
|
|
||||||
|
if i > len(words) - 4:
|
||||||
|
break
|
|
@ -1,16 +1,15 @@
|
||||||
"""Manipulate Markov data via the Django site."""
|
"""Manipulate Markov data via the Django site."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import random
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from django.contrib.auth.decorators import permission_required
|
from django.contrib.auth.decorators import permission_required
|
||||||
from django.db.models import Sum
|
|
||||||
from django.http import HttpResponse
|
from django.http import HttpResponse
|
||||||
from django.shortcuts import get_object_or_404, render
|
from django.shortcuts import get_object_or_404, render
|
||||||
|
|
||||||
from markov.forms import LogUploadForm, TeachLineForm
|
from markov.forms import LogUploadForm, TeachLineForm
|
||||||
from markov.models import MarkovContext, MarkovTarget, MarkovState
|
import markov.lib as markovlib
|
||||||
|
from markov.models import MarkovContext
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger('markov.views')
|
log = logging.getLogger('markov.views')
|
||||||
|
@ -27,7 +26,7 @@ def context_index(request, context_id):
|
||||||
|
|
||||||
start_t = time.time()
|
start_t = time.time()
|
||||||
context = get_object_or_404(MarkovContext, pk=context_id)
|
context = get_object_or_404(MarkovContext, pk=context_id)
|
||||||
chain = u" ".join(_generate_line(context))
|
chain = u" ".join(markovlib.generate_line(context))
|
||||||
end_t = time.time()
|
end_t = time.time()
|
||||||
|
|
||||||
return render(request, 'context.html', {'chain': chain,
|
return render(request, 'context.html', {'chain': chain,
|
||||||
|
@ -66,7 +65,7 @@ def import_file(request):
|
||||||
|
|
||||||
# this is a line we probably care about now
|
# this is a line we probably care about now
|
||||||
what = [x for x in what.rstrip().split(' ') if x not in strips]
|
what = [x for x in what.rstrip().split(' ') if x not in strips]
|
||||||
_learn_line(' '.join(what), context)
|
markovlib.learn_line(' '.join(what), context)
|
||||||
|
|
||||||
log.debug(set(whos))
|
log.debug(set(whos))
|
||||||
else:
|
else:
|
||||||
|
@ -86,133 +85,8 @@ def teach_line(request):
|
||||||
context = form.cleaned_data['context']
|
context = form.cleaned_data['context']
|
||||||
strips = form.cleaned_data['strip_prefixes'].split(' ')
|
strips = form.cleaned_data['strip_prefixes'].split(' ')
|
||||||
what = [x for x in line.rstrip().split(' ') if x not in strips]
|
what = [x for x in line.rstrip().split(' ') if x not in strips]
|
||||||
_learn_line(' '.join(what), context)
|
markovlib.learn_line(' '.join(what), context)
|
||||||
else:
|
else:
|
||||||
form = TeachLineForm()
|
form = TeachLineForm()
|
||||||
|
|
||||||
return render(request, 'teach_line.html', {'form': form})
|
return render(request, 'teach_line.html', {'form': form})
|
||||||
|
|
||||||
|
|
||||||
def _generate_sentence(context, topics=None, max_words=30):
|
|
||||||
"""Generate a Markov chain."""
|
|
||||||
|
|
||||||
words = []
|
|
||||||
# if we have topics, try to work from it and work backwards
|
|
||||||
if topics:
|
|
||||||
topic_word = random.choice(topics)
|
|
||||||
topics.remove(topic_word)
|
|
||||||
log.debug(u"looking for topic '{0:s}'".format(topic_word))
|
|
||||||
new_states = MarkovState.objects.filter(context=context, v=topic_word)
|
|
||||||
|
|
||||||
if len(new_states) > 0:
|
|
||||||
log.debug(u"found '{0:s}', starting backwards".format(topic_word))
|
|
||||||
words.insert(0, topic_word)
|
|
||||||
while len(words) <= max_words and words[0] != MarkovState._start2:
|
|
||||||
log.debug(u"looking backwards for '{0:s}'".format(words[0]))
|
|
||||||
new_states = MarkovState.objects.filter(context=context, v=words[0])
|
|
||||||
words.insert(0, _get_word_out_of_states(new_states, backwards=True))
|
|
||||||
|
|
||||||
# if we didn't get topic stuff, we need to start (forwards) here
|
|
||||||
if len(words) == 0:
|
|
||||||
words = [MarkovState._start1, MarkovState._start2]
|
|
||||||
|
|
||||||
i = len(words)
|
|
||||||
while len(words) <= max_words and words[-1] != MarkovState._stop:
|
|
||||||
log.debug(u"looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
|
|
||||||
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
|
|
||||||
log.debug(u"states retrieved")
|
|
||||||
words.append(_get_word_out_of_states(new_states))
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
words = [word for word in words if word not in
|
|
||||||
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
|
|
||||||
|
|
||||||
return words
|
|
||||||
|
|
||||||
|
|
||||||
def _generate_longish_sentence(context, topics=None, min_words=4, max_words=30):
|
|
||||||
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
|
|
||||||
|
|
||||||
tries = 0
|
|
||||||
while tries < 5:
|
|
||||||
sent = _generate_sentence(context, topics=topics, max_words=max_words)
|
|
||||||
if len(sent) >= min_words:
|
|
||||||
return sent
|
|
||||||
|
|
||||||
tries += 1
|
|
||||||
|
|
||||||
# if we got here, we need to just give up
|
|
||||||
return _generate_sentence(context)
|
|
||||||
|
|
||||||
|
|
||||||
def _generate_line(context, topics=None, min_words=15, max_words=30, max_sentences=3):
|
|
||||||
"""String multiple sentences together into a coherent sentence."""
|
|
||||||
|
|
||||||
tries = 0
|
|
||||||
sentences = 0
|
|
||||||
line = []
|
|
||||||
while tries < 5:
|
|
||||||
line += _generate_longish_sentence(context, topics=topics, max_words=max_words)
|
|
||||||
sentences += 1
|
|
||||||
if sentences >= max_sentences:
|
|
||||||
return line
|
|
||||||
if len(line) >= min_words:
|
|
||||||
return line
|
|
||||||
else:
|
|
||||||
if line[-1][-1] not in [',', '.', '!']:
|
|
||||||
line[-1] += random.choice([',', '.', '!'])
|
|
||||||
|
|
||||||
tries += 1
|
|
||||||
|
|
||||||
# if we got here, we need to give up
|
|
||||||
return line
|
|
||||||
|
|
||||||
|
|
||||||
def _get_word_out_of_states(states, backwards=False):
|
|
||||||
"""Pick one random word out of the given states."""
|
|
||||||
|
|
||||||
new_word = ''
|
|
||||||
running = 0
|
|
||||||
count_sum = states.aggregate(Sum('count'))['count__sum']
|
|
||||||
hit = random.randint(0, count_sum)
|
|
||||||
|
|
||||||
log.debug(u"sum: {0:d} hit: {1:d}".format(count_sum, hit))
|
|
||||||
|
|
||||||
states_itr = states.iterator()
|
|
||||||
for state in states_itr:
|
|
||||||
running += state.count
|
|
||||||
if running >= hit:
|
|
||||||
if backwards:
|
|
||||||
new_word = state.k2
|
|
||||||
else:
|
|
||||||
new_word = state.v
|
|
||||||
|
|
||||||
break
|
|
||||||
|
|
||||||
log.debug(u"found '{0:s}'".format(new_word))
|
|
||||||
return new_word
|
|
||||||
|
|
||||||
|
|
||||||
def _learn_line(line, context):
|
|
||||||
"""Create a bunch of MarkovStates for a given line of text."""
|
|
||||||
|
|
||||||
log.debug(u"learning {0:.40s}...".format(line))
|
|
||||||
|
|
||||||
words = line.split()
|
|
||||||
words = [MarkovState._start1, MarkovState._start2] + words + [MarkovState._stop]
|
|
||||||
|
|
||||||
for word in words:
|
|
||||||
if len(word) > MarkovState._meta.get_field('k1').max_length:
|
|
||||||
return
|
|
||||||
|
|
||||||
for i, word in enumerate(words):
|
|
||||||
log.debug(u"'{0:s}','{1:s}' -> '{2:s}'".format(words[i], words[i+1], words[i+2]))
|
|
||||||
state, created = MarkovState.objects.get_or_create(context=context,
|
|
||||||
k1=words[i],
|
|
||||||
k2=words[i+1],
|
|
||||||
v=words[i+2])
|
|
||||||
state.count += 1
|
|
||||||
state.save()
|
|
||||||
|
|
||||||
if i > len(words) - 4:
|
|
||||||
break
|
|
||||||
|
|
|
@ -25,8 +25,8 @@ import time
|
||||||
|
|
||||||
from dateutil.relativedelta import relativedelta
|
from dateutil.relativedelta import relativedelta
|
||||||
|
|
||||||
|
import markov.lib as markovlib
|
||||||
from markov.models import MarkovContext, MarkovState, MarkovTarget
|
from markov.models import MarkovContext, MarkovState, MarkovTarget
|
||||||
from markov.views import _generate_line, _learn_line
|
|
||||||
|
|
||||||
from extlib import irclib
|
from extlib import irclib
|
||||||
|
|
||||||
|
@ -66,10 +66,6 @@ class Markov(Module):
|
||||||
self.next_chatter_check = 0
|
self.next_chatter_check = 0
|
||||||
thread.start_new_thread(self.thread_do, ())
|
thread.start_new_thread(self.thread_do, ())
|
||||||
|
|
||||||
# TODO: bring this back somehow
|
|
||||||
#irc.xmlrpc_register_function(self._generate_line,
|
|
||||||
# "markov_generate_line")
|
|
||||||
|
|
||||||
def register_handlers(self):
|
def register_handlers(self):
|
||||||
"""Handle pubmsg/privmsg, to learn and/or reply to IRC events."""
|
"""Handle pubmsg/privmsg, to learn and/or reply to IRC events."""
|
||||||
|
|
||||||
|
@ -111,8 +107,8 @@ class Markov(Module):
|
||||||
return
|
return
|
||||||
|
|
||||||
if not event._recursing:
|
if not event._recursing:
|
||||||
context = _get_or_create_target_context(target)
|
context = markovlib.get_or_create_target_context(target)
|
||||||
_learn_line(what, context)
|
markovlib.learn_line(what, context)
|
||||||
|
|
||||||
def do(self, connection, event, nick, userhost, what, admin_unlocked):
|
def do(self, connection, event, nick, userhost, what, admin_unlocked):
|
||||||
"""Handle commands and inputs."""
|
"""Handle commands and inputs."""
|
||||||
|
@ -129,7 +125,7 @@ class Markov(Module):
|
||||||
if not self.shut_up:
|
if not self.shut_up:
|
||||||
# not a command, so see if i'm being mentioned
|
# not a command, so see if i'm being mentioned
|
||||||
if re.search(connection.get_nickname(), what, re.IGNORECASE) is not None:
|
if re.search(connection.get_nickname(), what, re.IGNORECASE) is not None:
|
||||||
context = _get_or_create_target_context(target)
|
context = markovlib.get_or_create_target_context(target)
|
||||||
|
|
||||||
addressed_pattern = '^' + connection.get_nickname() + '[:,]\s+(.*)'
|
addressed_pattern = '^' + connection.get_nickname() + '[:,]\s+(.*)'
|
||||||
addressed_re = re.compile(addressed_pattern)
|
addressed_re = re.compile(addressed_pattern)
|
||||||
|
@ -140,15 +136,15 @@ class Markov(Module):
|
||||||
|
|
||||||
self.lines_seen.append(('.self.said.', datetime.now()))
|
self.lines_seen.append(('.self.said.', datetime.now()))
|
||||||
return self.irc.reply(event, u"{0:s}: {1:s}".format(nick,
|
return self.irc.reply(event, u"{0:s}: {1:s}".format(nick,
|
||||||
u" ".join(_generate_line(context, topics=topics, max_sentences=1))))
|
u" ".join(markovlib.generate_line(context, topics=topics, max_sentences=1))))
|
||||||
else:
|
else:
|
||||||
# i wasn't addressed directly, so just respond
|
# i wasn't addressed directly, so just respond
|
||||||
topics = [x for x in what.split(' ') if len(x) >= 3]
|
topics = [x for x in what.split(' ') if len(x) >= 3]
|
||||||
self.lines_seen.append(('.self.said.', datetime.now()))
|
self.lines_seen.append(('.self.said.', datetime.now()))
|
||||||
|
|
||||||
return self.irc.reply(event, u"{0:s}".format(u" ".join(_generate_line(context,
|
return self.irc.reply(event, u"{0:s}".format(u" ".join(markovlib.generate_line(context,
|
||||||
topics=topics,
|
topics=topics,
|
||||||
max_sentences=1))))
|
max_sentences=1))))
|
||||||
|
|
||||||
def markov_learn(self, event, nick, userhost, what, admin_unlocked):
|
def markov_learn(self, event, nick, userhost, what, admin_unlocked):
|
||||||
"""Learn one line, as provided to the command."""
|
"""Learn one line, as provided to the command."""
|
||||||
|
@ -161,8 +157,8 @@ class Markov(Module):
|
||||||
match = self.learnre.search(what)
|
match = self.learnre.search(what)
|
||||||
if match:
|
if match:
|
||||||
line = match.group(1)
|
line = match.group(1)
|
||||||
context = _get_or_create_target_context(target)
|
context = markovlib.get_or_create_target_context(target)
|
||||||
_learn_line(line, context)
|
markovlib.learn_line(line, context)
|
||||||
|
|
||||||
# return what was learned, for weird chaining purposes
|
# return what was learned, for weird chaining purposes
|
||||||
return line
|
return line
|
||||||
|
@ -179,7 +175,7 @@ class Markov(Module):
|
||||||
if match:
|
if match:
|
||||||
min_size = 15
|
min_size = 15
|
||||||
max_size = 30
|
max_size = 30
|
||||||
context = _get_or_create_target_context(target)
|
context = markovlib.get_or_create_target_context(target)
|
||||||
|
|
||||||
if match.group(2):
|
if match.group(2):
|
||||||
min_size = int(match.group(2))
|
min_size = int(match.group(2))
|
||||||
|
@ -191,14 +187,14 @@ class Markov(Module):
|
||||||
topics = [x for x in line.split(' ') if len(x) >= 3]
|
topics = [x for x in line.split(' ') if len(x) >= 3]
|
||||||
|
|
||||||
self.lines_seen.append(('.self.said.', datetime.now()))
|
self.lines_seen.append(('.self.said.', datetime.now()))
|
||||||
return u" ".join(_generate_line(context, topics=topics,
|
return u" ".join(markovlib.generate_line(context, topics=topics,
|
||||||
min_words=min_size, max_words=max_size,
|
min_words=min_size, max_words=max_size,
|
||||||
max_sentences=1))
|
max_sentences=1))
|
||||||
else:
|
else:
|
||||||
self.lines_seen.append(('.self.said.', datetime.now()))
|
self.lines_seen.append(('.self.said.', datetime.now()))
|
||||||
return u" ".join(_generate_line(context, min_words=min_size,
|
return u" ".join(markovlib.generate_line(context, min_words=min_size,
|
||||||
max_words=max_size,
|
max_words=max_size,
|
||||||
max_sentences=1))
|
max_sentences=1))
|
||||||
|
|
||||||
def thread_do(self):
|
def thread_do(self):
|
||||||
"""Do various things."""
|
"""Do various things."""
|
||||||
|
@ -237,35 +233,3 @@ class Markov(Module):
|
||||||
for t in targets:
|
for t in targets:
|
||||||
self.sendmsg(t['target'],
|
self.sendmsg(t['target'],
|
||||||
'shutting up for 30 seconds due to last 30 seconds of activity')
|
'shutting up for 30 seconds due to last 30 seconds of activity')
|
||||||
|
|
||||||
def _get_or_create_target_context(target_name):
|
|
||||||
"""Return the context for a provided nick/channel, creating missing ones."""
|
|
||||||
|
|
||||||
# find the stuff, or create it
|
|
||||||
try:
|
|
||||||
target = MarkovTarget.objects.get(name=target_name)
|
|
||||||
return target.context
|
|
||||||
except MarkovContext.DoesNotExist:
|
|
||||||
# make a context
|
|
||||||
context = MarkovContext()
|
|
||||||
context.name = target_name
|
|
||||||
context.save()
|
|
||||||
|
|
||||||
target.context = context
|
|
||||||
target.save()
|
|
||||||
|
|
||||||
return target.context
|
|
||||||
except MarkovTarget.DoesNotExist:
|
|
||||||
# first we need to make a context for this
|
|
||||||
context = MarkovContext()
|
|
||||||
context.name = target_name
|
|
||||||
context.save()
|
|
||||||
|
|
||||||
target = MarkovTarget()
|
|
||||||
target.name = target_name
|
|
||||||
target.context = context
|
|
||||||
target.save()
|
|
||||||
|
|
||||||
return target.context
|
|
||||||
|
|
||||||
# vi:tabstop=4:expandtab:autoindent
|
|
||||||
|
|
Loading…
Reference in New Issue