Merge branch 'markov-tweaks' into 'master'

Markov tweaks: new sentence generator, new import

This changes the way that sentences are generated, to ideally be a bit faster than usual, and also adds another import method that just adds text, rather than assuming IRC logs.

See merge request !9
This commit is contained in:
Brian S. Stephan 2016-12-04 10:54:06 -06:00
commit 1b8faaca9e
6 changed files with 144 additions and 53 deletions

View File

@ -19,7 +19,7 @@ admin.site.register(MarkovTarget)
admin.site.register(MarkovState)
@permission_required('import_log_file', raise_exception=True)
@permission_required('import_text_file', raise_exception=True)
def import_file(request):
"""Accept a file upload and turn it into markov stuff.
@ -30,31 +30,58 @@ def import_file(request):
if request.method == 'POST':
form = LogUploadForm(request.POST, request.FILES)
if form.is_valid():
log_file = request.FILES['log_file']
context = form.cleaned_data['context']
ignores = form.cleaned_data['ignore_nicks'].split(',')
strips = form.cleaned_data['strip_prefixes'].split(' ')
if form.cleaned_data['text_file_format'] == LogUploadForm.FILE_FORMAT_WEECHAT:
text_file = request.FILES['text_file']
context = form.cleaned_data['context']
ignores = form.cleaned_data['ignore_nicks'].split(',')
strips = form.cleaned_data['strip_prefixes'].split(' ')
whos = []
for line in log_file:
log.debug(line)
(timestamp, who, what) = line.decode('utf-8').split('\t', 2)
whos = []
for line in text_file:
log.debug(line)
(timestamp, who, what) = line.decode('utf-8').split('\t', 2)
if who in ('-->', '<--', '--', ' *'):
continue
if who in ('-->', '<--', '--', ' *'):
continue
if who in ignores:
continue
if who in ignores:
continue
whos.append(who)
whos.append(who)
# this is a line we probably care about now
what = [x for x in what.rstrip().split(' ') if x not in strips]
markovlib.learn_line(' '.join(what), context)
log.debug("learned")
# this is a line we probably care about now
what = [x for x in what.rstrip().split(' ') if x not in strips]
markovlib.learn_line(' '.join(what), context)
log.debug("learned")
log.debug(set(whos))
form = LogUploadForm()
log.debug(set(whos))
form = LogUploadForm()
elif form.cleaned_data['text_file_format'] == LogUploadForm.FILE_FORMAT_RAW_TEXT:
text_file = request.FILES['text_file']
context = form.cleaned_data['context']
k1 = MarkovState._start1
k2 = MarkovState._start2
for line in text_file:
for word in [x for x in line.decode('utf-8') .rstrip().split(' ')]:
log.info(word)
if word:
state, created = MarkovState.objects.get_or_create(context=context, k1=k1,
k2=k2, v=word)
state.count += 1
state.save()
if word[-1] in ['.', '?', '!']:
state, created = MarkovState.objects.get_or_create(context=context, k1=k2,
k2=word, v=MarkovState._stop)
state.count += 1
state.save()
k1 = MarkovState._start1
k2 = MarkovState._start2
else:
k1 = k2
k2 = word
else:
form = LogUploadForm()

View File

@ -2,7 +2,7 @@
import logging
from django.forms import Form, CharField, FileField, ModelChoiceField
from django.forms import Form, CharField, ChoiceField, FileField, ModelChoiceField
from markov.models import MarkovContext
@ -13,11 +13,20 @@ class LogUploadForm(Form):
"""Accept a file upload that will be imported into Markov stuff."""
log_file = FileField(help_text="Weechat log format.")
FILE_FORMAT_WEECHAT = 'WEECHAT'
FILE_FORMAT_RAW_TEXT = 'RAW'
FILE_FORMAT_CHOICES = (
(FILE_FORMAT_WEECHAT, "Weechat"),
(FILE_FORMAT_RAW_TEXT, "Raw text file"),
)
text_file = FileField()
text_file_format = ChoiceField(choices=FILE_FORMAT_CHOICES)
context = ModelChoiceField(queryset=MarkovContext.objects.all())
ignore_nicks = CharField(help_text="Comma-separated list of nicks to ignore.",
ignore_nicks = CharField(help_text="Comma-separated list of nicks to ignore. For Weechat logs.",
required=False)
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip.",
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip. For Weechat logs.",
required=False)

View File

@ -55,12 +55,10 @@ class Markov(Plugin):
topics = [x for x in line.split(' ') if len(x) >= 3]
return self.bot.reply(event, " ".join(markovlib.generate_line(context, topics=topics,
min_words=min_size, max_words=max_size,
max_sentences=1)))
min_words=min_size, max_words=max_size)))
else:
return self.bot.reply(event, " ".join(markovlib.generate_line(context, min_words=min_size,
max_words=max_size,
max_sentences=1)))
max_words=max_size)))
def handle_chatter(self, connection, event):
"""Learn from IRC chatter."""
@ -98,17 +96,13 @@ class Markov(Plugin):
topics = [x for x in addressed_re.match(what).group(1).split(' ') if len(x) >= 3]
return self.bot.reply(event, "{0:s}: {1:s}"
"".format(nick, " ".join(markovlib.generate_line(context,
topics=topics,
max_sentences=1))))
"".format(nick, " ".join(markovlib.generate_line(context, topics=topics))))
else:
# i wasn't addressed directly, so just respond
topics = [x for x in what.split(' ') if len(x) >= 3]
return self.bot.reply(event, "{0:s}"
"".format(" ".join(markovlib.generate_line(context,
topics=topics,
max_sentences=1))))
"".format(" ".join(markovlib.generate_line(context, topics=topics))))
plugin = Markov

View File

@ -9,22 +9,21 @@ from markov.models import MarkovContext, MarkovState, MarkovTarget
log = logging.getLogger('markov.lib')
def generate_line(context, topics=None, min_words=15, max_words=30, max_sentences=3):
def generate_line(context, topics=None, min_words=15, max_words=30, sentence_bias=2, max_tries=5):
"""String multiple sentences together into a coherent sentence."""
tries = 0
sentences = 0
line = []
while tries < 5:
line += generate_longish_sentence(context, topics=topics, max_words=max_words)
sentences += 1
if sentences >= max_sentences:
return line
min_words_per_sentence = min_words / sentence_bias
while tries < max_tries:
line += generate_longish_sentence(context, topics=topics, min_words=min_words_per_sentence,
max_words=max_words, max_tries=max_tries)
if len(line) >= min_words:
return line
else:
if line[-1][-1] not in [',', '.', '!']:
line[-1] += random.choice([',', '.', '!'])
if len(line) > 0:
if line[-1][-1] not in [',', '.', '!', '?', ':']:
line[-1] += random.choice(['?', '.', '!'])
tries += 1
@ -32,22 +31,26 @@ def generate_line(context, topics=None, min_words=15, max_words=30, max_sentence
return line
def generate_longish_sentence(context, topics=None, min_words=4, max_words=30):
def generate_longish_sentence(context, topics=None, min_words=15, max_words=30, max_tries=100):
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
sent = ""
tries = 0
while tries < 5:
sent = generate_sentence(context, topics=topics, max_words=max_words)
while tries < max_tries:
sent = generate_sentence(context, topics=topics, min_words=min_words, max_words=max_words)
if len(sent) >= min_words:
log.debug("found a longish sentence, %s", sent)
return sent
else:
log.debug("%s isn't long enough, going to try again", sent)
tries += 1
# if we got here, we need to just give up
return generate_sentence(context)
return sent
def generate_sentence(context, topics=None, max_words=30):
def generate_sentence(context, topics=None, min_words=15, max_words=30):
"""Generate a Markov chain."""
words = []
@ -64,23 +67,63 @@ def generate_sentence(context, topics=None, max_words=30):
while len(words) <= max_words and words[0] != MarkovState._start2:
log.debug("looking backwards for '{0:s}'".format(words[0]))
new_states = MarkovState.objects.filter(context=context, v=words[0])
words.insert(0, get_word_out_of_states(new_states, backwards=True))
# if we find a start, use it
if MarkovState._start2 in new_states:
log.debug("found a start2 in the results, intentionally picking it")
words.insert(0, MarkovState._start2)
else:
words.insert(0, get_word_out_of_states(new_states, backwards=True))
log.debug("picked %s", words[0])
# if we didn't get topic stuff, we need to start (forwards) here
# if what we found is too long, abandon it, sadly
if len(words) > max_words:
log.debug("%s is too long, i'm going to give up on it", words)
words.clear()
# if we didn't get topic stuff, we need to start (forwards) here, otherwise we use
# what we already put together (obviously)
if len(words) == 0:
words = [MarkovState._start1, MarkovState._start2]
i = len(words)
while len(words) <= max_words and words[-1] != MarkovState._stop:
while words[-1] != MarkovState._stop:
log.debug("looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
log.debug("states retrieved")
words.append(get_word_out_of_states(new_states))
# try to find states that are in our targets
if topics and len(topics):
target_hits = list(set(words).intersection(set(topics)))
else:
target_hits = []
if len(words) > min_words and MarkovState._stop in new_states:
# if we're over min_words, and got a stop naturally, use it
log.debug("found a stop in the results, intentionally picking it")
words.append(MarkovState._stop)
elif len(target_hits) > 0:
# if there's a target word in the states, pick it
target_hit = random.choice(target_hits)
log.debug("found a topic hit %s, using it", target_hit)
topics.remove(target_hit)
words.append(target_hit)
elif len(words) <= min_words:
# if we still need more words, intentionally avoid stop
words.append(get_word_out_of_states(new_states.exclude(v=MarkovState._stop)))
log.debug("picked (stop avoidance) %s", words[-1])
else:
words.append(get_word_out_of_states(new_states))
log.debug("picked %s", words[-1])
i += 1
words = [word for word in words if word not in
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
# if what we found is too long, abandon it, sadly
if len(words) > max_words:
log.debug("%s is too long, i'm going to give up on it", words)
words.clear()
return words

View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('markov', '0002_auto_20150514_2317'),
]
operations = [
migrations.AlterModelOptions(
name='markovstate',
options={'permissions': set([('import_text_file', 'Can import states from a text file'), ('teach_line', 'Can teach lines')])},
),
]

View File

@ -59,7 +59,7 @@ class MarkovState(models.Model):
['context', 'v'],
]
permissions = {
('import_log_file', "Can import states from a log file"),
('import_text_file', "Can import states from a text file"),
('teach_line', "Can teach lines"),
}
unique_together = ('context', 'k1', 'k2', 'v')