Merge branch 'markov-tweaks' into 'master'
Markov tweaks: new sentence generator, new import This changes the way that sentences are generated, to ideally be a bit faster than usual, and also adds another import method that just adds text, rather than assuming IRC logs. See merge request !9
This commit is contained in:
commit
1b8faaca9e
|
@ -19,7 +19,7 @@ admin.site.register(MarkovTarget)
|
||||||
admin.site.register(MarkovState)
|
admin.site.register(MarkovState)
|
||||||
|
|
||||||
|
|
||||||
@permission_required('import_log_file', raise_exception=True)
|
@permission_required('import_text_file', raise_exception=True)
|
||||||
def import_file(request):
|
def import_file(request):
|
||||||
"""Accept a file upload and turn it into markov stuff.
|
"""Accept a file upload and turn it into markov stuff.
|
||||||
|
|
||||||
|
@ -30,31 +30,58 @@ def import_file(request):
|
||||||
if request.method == 'POST':
|
if request.method == 'POST':
|
||||||
form = LogUploadForm(request.POST, request.FILES)
|
form = LogUploadForm(request.POST, request.FILES)
|
||||||
if form.is_valid():
|
if form.is_valid():
|
||||||
log_file = request.FILES['log_file']
|
if form.cleaned_data['text_file_format'] == LogUploadForm.FILE_FORMAT_WEECHAT:
|
||||||
context = form.cleaned_data['context']
|
text_file = request.FILES['text_file']
|
||||||
ignores = form.cleaned_data['ignore_nicks'].split(',')
|
context = form.cleaned_data['context']
|
||||||
strips = form.cleaned_data['strip_prefixes'].split(' ')
|
ignores = form.cleaned_data['ignore_nicks'].split(',')
|
||||||
|
strips = form.cleaned_data['strip_prefixes'].split(' ')
|
||||||
|
|
||||||
whos = []
|
whos = []
|
||||||
for line in log_file:
|
for line in text_file:
|
||||||
log.debug(line)
|
log.debug(line)
|
||||||
(timestamp, who, what) = line.decode('utf-8').split('\t', 2)
|
(timestamp, who, what) = line.decode('utf-8').split('\t', 2)
|
||||||
|
|
||||||
if who in ('-->', '<--', '--', ' *'):
|
if who in ('-->', '<--', '--', ' *'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if who in ignores:
|
if who in ignores:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
whos.append(who)
|
whos.append(who)
|
||||||
|
|
||||||
# this is a line we probably care about now
|
# this is a line we probably care about now
|
||||||
what = [x for x in what.rstrip().split(' ') if x not in strips]
|
what = [x for x in what.rstrip().split(' ') if x not in strips]
|
||||||
markovlib.learn_line(' '.join(what), context)
|
markovlib.learn_line(' '.join(what), context)
|
||||||
log.debug("learned")
|
log.debug("learned")
|
||||||
|
|
||||||
log.debug(set(whos))
|
log.debug(set(whos))
|
||||||
form = LogUploadForm()
|
form = LogUploadForm()
|
||||||
|
elif form.cleaned_data['text_file_format'] == LogUploadForm.FILE_FORMAT_RAW_TEXT:
|
||||||
|
text_file = request.FILES['text_file']
|
||||||
|
context = form.cleaned_data['context']
|
||||||
|
|
||||||
|
k1 = MarkovState._start1
|
||||||
|
k2 = MarkovState._start2
|
||||||
|
for line in text_file:
|
||||||
|
for word in [x for x in line.decode('utf-8') .rstrip().split(' ')]:
|
||||||
|
log.info(word)
|
||||||
|
if word:
|
||||||
|
state, created = MarkovState.objects.get_or_create(context=context, k1=k1,
|
||||||
|
k2=k2, v=word)
|
||||||
|
state.count += 1
|
||||||
|
state.save()
|
||||||
|
|
||||||
|
if word[-1] in ['.', '?', '!']:
|
||||||
|
state, created = MarkovState.objects.get_or_create(context=context, k1=k2,
|
||||||
|
k2=word, v=MarkovState._stop)
|
||||||
|
state.count += 1
|
||||||
|
state.save()
|
||||||
|
|
||||||
|
k1 = MarkovState._start1
|
||||||
|
k2 = MarkovState._start2
|
||||||
|
else:
|
||||||
|
k1 = k2
|
||||||
|
k2 = word
|
||||||
else:
|
else:
|
||||||
form = LogUploadForm()
|
form = LogUploadForm()
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from django.forms import Form, CharField, FileField, ModelChoiceField
|
from django.forms import Form, CharField, ChoiceField, FileField, ModelChoiceField
|
||||||
|
|
||||||
from markov.models import MarkovContext
|
from markov.models import MarkovContext
|
||||||
|
|
||||||
|
@ -13,11 +13,20 @@ class LogUploadForm(Form):
|
||||||
|
|
||||||
"""Accept a file upload that will be imported into Markov stuff."""
|
"""Accept a file upload that will be imported into Markov stuff."""
|
||||||
|
|
||||||
log_file = FileField(help_text="Weechat log format.")
|
FILE_FORMAT_WEECHAT = 'WEECHAT'
|
||||||
|
FILE_FORMAT_RAW_TEXT = 'RAW'
|
||||||
|
|
||||||
|
FILE_FORMAT_CHOICES = (
|
||||||
|
(FILE_FORMAT_WEECHAT, "Weechat"),
|
||||||
|
(FILE_FORMAT_RAW_TEXT, "Raw text file"),
|
||||||
|
)
|
||||||
|
|
||||||
|
text_file = FileField()
|
||||||
|
text_file_format = ChoiceField(choices=FILE_FORMAT_CHOICES)
|
||||||
context = ModelChoiceField(queryset=MarkovContext.objects.all())
|
context = ModelChoiceField(queryset=MarkovContext.objects.all())
|
||||||
ignore_nicks = CharField(help_text="Comma-separated list of nicks to ignore.",
|
ignore_nicks = CharField(help_text="Comma-separated list of nicks to ignore. For Weechat logs.",
|
||||||
required=False)
|
required=False)
|
||||||
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip.",
|
strip_prefixes = CharField(help_text="Space-separated list of line prefixes to strip. For Weechat logs.",
|
||||||
required=False)
|
required=False)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -55,12 +55,10 @@ class Markov(Plugin):
|
||||||
topics = [x for x in line.split(' ') if len(x) >= 3]
|
topics = [x for x in line.split(' ') if len(x) >= 3]
|
||||||
|
|
||||||
return self.bot.reply(event, " ".join(markovlib.generate_line(context, topics=topics,
|
return self.bot.reply(event, " ".join(markovlib.generate_line(context, topics=topics,
|
||||||
min_words=min_size, max_words=max_size,
|
min_words=min_size, max_words=max_size)))
|
||||||
max_sentences=1)))
|
|
||||||
else:
|
else:
|
||||||
return self.bot.reply(event, " ".join(markovlib.generate_line(context, min_words=min_size,
|
return self.bot.reply(event, " ".join(markovlib.generate_line(context, min_words=min_size,
|
||||||
max_words=max_size,
|
max_words=max_size)))
|
||||||
max_sentences=1)))
|
|
||||||
|
|
||||||
def handle_chatter(self, connection, event):
|
def handle_chatter(self, connection, event):
|
||||||
"""Learn from IRC chatter."""
|
"""Learn from IRC chatter."""
|
||||||
|
@ -98,17 +96,13 @@ class Markov(Plugin):
|
||||||
topics = [x for x in addressed_re.match(what).group(1).split(' ') if len(x) >= 3]
|
topics = [x for x in addressed_re.match(what).group(1).split(' ') if len(x) >= 3]
|
||||||
|
|
||||||
return self.bot.reply(event, "{0:s}: {1:s}"
|
return self.bot.reply(event, "{0:s}: {1:s}"
|
||||||
"".format(nick, " ".join(markovlib.generate_line(context,
|
"".format(nick, " ".join(markovlib.generate_line(context, topics=topics))))
|
||||||
topics=topics,
|
|
||||||
max_sentences=1))))
|
|
||||||
else:
|
else:
|
||||||
# i wasn't addressed directly, so just respond
|
# i wasn't addressed directly, so just respond
|
||||||
topics = [x for x in what.split(' ') if len(x) >= 3]
|
topics = [x for x in what.split(' ') if len(x) >= 3]
|
||||||
|
|
||||||
return self.bot.reply(event, "{0:s}"
|
return self.bot.reply(event, "{0:s}"
|
||||||
"".format(" ".join(markovlib.generate_line(context,
|
"".format(" ".join(markovlib.generate_line(context, topics=topics))))
|
||||||
topics=topics,
|
|
||||||
max_sentences=1))))
|
|
||||||
|
|
||||||
|
|
||||||
plugin = Markov
|
plugin = Markov
|
||||||
|
|
|
@ -9,22 +9,21 @@ from markov.models import MarkovContext, MarkovState, MarkovTarget
|
||||||
log = logging.getLogger('markov.lib')
|
log = logging.getLogger('markov.lib')
|
||||||
|
|
||||||
|
|
||||||
def generate_line(context, topics=None, min_words=15, max_words=30, max_sentences=3):
|
def generate_line(context, topics=None, min_words=15, max_words=30, sentence_bias=2, max_tries=5):
|
||||||
"""String multiple sentences together into a coherent sentence."""
|
"""String multiple sentences together into a coherent sentence."""
|
||||||
|
|
||||||
tries = 0
|
tries = 0
|
||||||
sentences = 0
|
|
||||||
line = []
|
line = []
|
||||||
while tries < 5:
|
min_words_per_sentence = min_words / sentence_bias
|
||||||
line += generate_longish_sentence(context, topics=topics, max_words=max_words)
|
while tries < max_tries:
|
||||||
sentences += 1
|
line += generate_longish_sentence(context, topics=topics, min_words=min_words_per_sentence,
|
||||||
if sentences >= max_sentences:
|
max_words=max_words, max_tries=max_tries)
|
||||||
return line
|
|
||||||
if len(line) >= min_words:
|
if len(line) >= min_words:
|
||||||
return line
|
return line
|
||||||
else:
|
else:
|
||||||
if line[-1][-1] not in [',', '.', '!']:
|
if len(line) > 0:
|
||||||
line[-1] += random.choice([',', '.', '!'])
|
if line[-1][-1] not in [',', '.', '!', '?', ':']:
|
||||||
|
line[-1] += random.choice(['?', '.', '!'])
|
||||||
|
|
||||||
tries += 1
|
tries += 1
|
||||||
|
|
||||||
|
@ -32,22 +31,26 @@ def generate_line(context, topics=None, min_words=15, max_words=30, max_sentence
|
||||||
return line
|
return line
|
||||||
|
|
||||||
|
|
||||||
def generate_longish_sentence(context, topics=None, min_words=4, max_words=30):
|
def generate_longish_sentence(context, topics=None, min_words=15, max_words=30, max_tries=100):
|
||||||
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
|
"""Generate a Markov chain, but throw away the short ones unless we get desperate."""
|
||||||
|
|
||||||
|
sent = ""
|
||||||
tries = 0
|
tries = 0
|
||||||
while tries < 5:
|
while tries < max_tries:
|
||||||
sent = generate_sentence(context, topics=topics, max_words=max_words)
|
sent = generate_sentence(context, topics=topics, min_words=min_words, max_words=max_words)
|
||||||
if len(sent) >= min_words:
|
if len(sent) >= min_words:
|
||||||
|
log.debug("found a longish sentence, %s", sent)
|
||||||
return sent
|
return sent
|
||||||
|
else:
|
||||||
|
log.debug("%s isn't long enough, going to try again", sent)
|
||||||
|
|
||||||
tries += 1
|
tries += 1
|
||||||
|
|
||||||
# if we got here, we need to just give up
|
# if we got here, we need to just give up
|
||||||
return generate_sentence(context)
|
return sent
|
||||||
|
|
||||||
|
|
||||||
def generate_sentence(context, topics=None, max_words=30):
|
def generate_sentence(context, topics=None, min_words=15, max_words=30):
|
||||||
"""Generate a Markov chain."""
|
"""Generate a Markov chain."""
|
||||||
|
|
||||||
words = []
|
words = []
|
||||||
|
@ -64,23 +67,63 @@ def generate_sentence(context, topics=None, max_words=30):
|
||||||
while len(words) <= max_words and words[0] != MarkovState._start2:
|
while len(words) <= max_words and words[0] != MarkovState._start2:
|
||||||
log.debug("looking backwards for '{0:s}'".format(words[0]))
|
log.debug("looking backwards for '{0:s}'".format(words[0]))
|
||||||
new_states = MarkovState.objects.filter(context=context, v=words[0])
|
new_states = MarkovState.objects.filter(context=context, v=words[0])
|
||||||
words.insert(0, get_word_out_of_states(new_states, backwards=True))
|
# if we find a start, use it
|
||||||
|
if MarkovState._start2 in new_states:
|
||||||
|
log.debug("found a start2 in the results, intentionally picking it")
|
||||||
|
words.insert(0, MarkovState._start2)
|
||||||
|
else:
|
||||||
|
words.insert(0, get_word_out_of_states(new_states, backwards=True))
|
||||||
|
log.debug("picked %s", words[0])
|
||||||
|
|
||||||
# if we didn't get topic stuff, we need to start (forwards) here
|
# if what we found is too long, abandon it, sadly
|
||||||
|
if len(words) > max_words:
|
||||||
|
log.debug("%s is too long, i'm going to give up on it", words)
|
||||||
|
words.clear()
|
||||||
|
|
||||||
|
# if we didn't get topic stuff, we need to start (forwards) here, otherwise we use
|
||||||
|
# what we already put together (obviously)
|
||||||
if len(words) == 0:
|
if len(words) == 0:
|
||||||
words = [MarkovState._start1, MarkovState._start2]
|
words = [MarkovState._start1, MarkovState._start2]
|
||||||
|
|
||||||
i = len(words)
|
i = len(words)
|
||||||
while len(words) <= max_words and words[-1] != MarkovState._stop:
|
while words[-1] != MarkovState._stop:
|
||||||
log.debug("looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
|
log.debug("looking for '{0:s}','{1:s}'".format(words[i-2], words[i-1]))
|
||||||
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
|
new_states = MarkovState.objects.filter(context=context, k1=words[i-2], k2=words[i-1])
|
||||||
log.debug("states retrieved")
|
log.debug("states retrieved")
|
||||||
words.append(get_word_out_of_states(new_states))
|
|
||||||
|
# try to find states that are in our targets
|
||||||
|
if topics and len(topics):
|
||||||
|
target_hits = list(set(words).intersection(set(topics)))
|
||||||
|
else:
|
||||||
|
target_hits = []
|
||||||
|
|
||||||
|
if len(words) > min_words and MarkovState._stop in new_states:
|
||||||
|
# if we're over min_words, and got a stop naturally, use it
|
||||||
|
log.debug("found a stop in the results, intentionally picking it")
|
||||||
|
words.append(MarkovState._stop)
|
||||||
|
elif len(target_hits) > 0:
|
||||||
|
# if there's a target word in the states, pick it
|
||||||
|
target_hit = random.choice(target_hits)
|
||||||
|
log.debug("found a topic hit %s, using it", target_hit)
|
||||||
|
topics.remove(target_hit)
|
||||||
|
words.append(target_hit)
|
||||||
|
elif len(words) <= min_words:
|
||||||
|
# if we still need more words, intentionally avoid stop
|
||||||
|
words.append(get_word_out_of_states(new_states.exclude(v=MarkovState._stop)))
|
||||||
|
log.debug("picked (stop avoidance) %s", words[-1])
|
||||||
|
else:
|
||||||
|
words.append(get_word_out_of_states(new_states))
|
||||||
|
log.debug("picked %s", words[-1])
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
words = [word for word in words if word not in
|
words = [word for word in words if word not in
|
||||||
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
|
(MarkovState._start1, MarkovState._start2, MarkovState._stop)]
|
||||||
|
|
||||||
|
# if what we found is too long, abandon it, sadly
|
||||||
|
if len(words) > max_words:
|
||||||
|
log.debug("%s is too long, i'm going to give up on it", words)
|
||||||
|
words.clear()
|
||||||
|
|
||||||
return words
|
return words
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('markov', '0002_auto_20150514_2317'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterModelOptions(
|
||||||
|
name='markovstate',
|
||||||
|
options={'permissions': set([('import_text_file', 'Can import states from a text file'), ('teach_line', 'Can teach lines')])},
|
||||||
|
),
|
||||||
|
]
|
|
@ -59,7 +59,7 @@ class MarkovState(models.Model):
|
||||||
['context', 'v'],
|
['context', 'v'],
|
||||||
]
|
]
|
||||||
permissions = {
|
permissions = {
|
||||||
('import_log_file', "Can import states from a log file"),
|
('import_text_file', "Can import states from a text file"),
|
||||||
('teach_line', "Can teach lines"),
|
('teach_line', "Can teach lines"),
|
||||||
}
|
}
|
||||||
unique_together = ('context', 'k1', 'k2', 'v')
|
unique_together = ('context', 'k1', 'k2', 'v')
|
||||||
|
|
Loading…
Reference in New Issue