Practical Deep Learning, Lesson 4, Language Model Blog Post Imitator

from fastai.text.all import *
from pathlib import Path

import warnings
warnings.filterwarnings('ignore')

---
<frontmatter key values>
---

<the rest of the post with code, links, images, shortcodes, etc.>

get_imdb = partial(get_text_files, folders=['train', 'test', 'unsup'])

dls_lm = DataBlock(
    blocks=TextBlock.from_folder(path, is_lm=True),
    get_items=get_imdb, splitter=RandomSplitter(0.1)
).dataloaders(path, path=path, bs=128, seq_len=80)

path = Path("./data/content")
files = get_files(path, extensions='.md', recurse=True)
for f in files[:3]:
    print(f)

data/content/posts/2013/2013-07-05-qc.md
data/content/posts/2024/models-writing-about-coding-with-models.md
data/content/posts/2024/vlms-hallucinate.md

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[209], line 10
      3 # First, create a tokenizer
      4 text_processor = TextBlock.from_folder(path, is_lm=True)
      6 dls_lm = DataBlock(
      7     blocks=text_processor,
      8     get_items=get,
      9     splitter=RandomSplitter(0.1)
---> 10 ).dataloaders(
     11     path,
     12     path=path,
     13     bs=128,
     14     seq_len=80,
     15 )

File ~/dev/lab/fastbook_projects/blog_post_generator/.venv/lib/python3.12/site-packages/fastai/data/block.py:157, in DataBlock.dataloaders(self, source, path, verbose, **kwargs)
    151 def dataloaders(self,
    152     source, # The data source
    153     path:str='.', # Data source and default `Learner` path
    154     verbose:bool=False, # Show verbose messages
    155     **kwargs
    156 ) -> DataLoaders:
--> 157     dsets = self.datasets(source, verbose=verbose)
    158     kwargs = {**self.dls_kwargs, **kwargs, 'verbose': verbose}
...
    387     self.types.append(type(x))
--> 388 types = L(t if is_listy(t) else [t] for t in self.types).concat().unique()
    389 self.pretty_types = '\n'.join([f'  - {t}' for t in types])

TypeError: 'NoneType' object is not iterable

    return f.read()
           ^^^^^^^^
  File "<frozen codecs>", line 322, in decode
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte

src_file.read_text(encoding=encoding)

import re


def clean_content(text):
    # Handle code blocks with language specifiers
    text = re.sub(r'```(\w+)?\n(.*?)```', lambda m: f'<CODE>{m.group(2)}</CODE>', text, flags=re.DOTALL)

    # Replace single backticks
    text = re.sub(r'`([^`]+)`', r'<INLINE_CODE>\1</INLINE_CODE>', text)

    return text

def copy_and_rename_files():
    src_dir = Path("./data/content")
    dst_dir = Path("./data/cleaned_content")

    if not dst_dir.exists():
        dst_dir.mkdir(parents=True)

    for src_file in src_dir.rglob("*.md"):
        try:
            content = None
            for encoding in ['utf-8', 'latin-1', 'cp1252']:
                try:
                    content = src_file.read_text(encoding=encoding)
                    if content.startswith('---'):
                        # Remove markdown frontmatter
                        parts = content.split('---', 2)
                        if len(parts) >= 3:
                            content = parts[2]
                    break
                except UnicodeDecodeError:
                    continue

            if content is None:
                print(f"Skipping {src_file}: Unable to decode with supported encodings")
                continue

            content = clean_content(content)

            rel_path = src_file.relative_to(src_dir)
            dst_file = dst_dir / rel_path.with_suffix('.txt')
            dst_file.parent.mkdir(parents=True, exist_ok=True)
            dst_file.write_text(content, encoding='utf-8')

        except Exception as e:
            print(f"Error processing {src_file}: {str(e)}")

copy_and_rename_files()

path = Path("./data/cleaned_content")
files = get_text_files(path)
for f in files[:3]:
    print(f)

data/cleaned_content/posts/2013/2013-07-05-qc.txt
data/cleaned_content/posts/2024/language-model-based-aggregators.txt
data/cleaned_content/posts/2024/making-your-vision-real.txt

get = partial(get_text_files, folders=['posts', 'til', 'logs', 'projects'])

text_processor = TextBlock.from_folder(path, is_lm=True)

dls_lm = DataBlock(
    blocks=text_processor,
    get_items=get,
    splitter=RandomSplitter(0.1)
).dataloaders(
    path,
    path=path,
    bs=128,
    seq_len=512,
)

dls_lm.show_batch(max_n=2)

learn = language_model_learner(
    dls_lm, AWD_LSTM, drop_mult=0.3,
    metrics=[accuracy, Perplexity()]
).to_fp16()

learn.fit_one_cycle(1, 2e-2)

learn.save('1epoch')

Path('data/cleaned_content/models/1epoch.pth')

learn = learn.load('1epoch')

learn.unfreeze()
learn.fit_one_cycle(10, 2e-3)

learn.save_encoder('finetuned')

TEXT = "I am interacting with a language model as a thought partner to"
N_WORDS = 256
TEMP = 0.75
pred = learn.predict(TEXT, N_WORDS, temperature=TEMP)

print(pred)

i am interacting with a language model as a thought partner to train .
 I think it should be more difficult to use Sonnet but it is often difficult to get to because i am a very familiar language model .

 In this way , i wanted to use the phrase Sonnet > rather than just see the phrase as a word .
 He 's also using the word " prompt " , which is a word word that is used in art , then as a title to describe a language model that extract structured data from an individual 's experience .
 I 've used this idea to describe how a language model has a structure with a single structure and i can try and solve this with the following following Sonnet code : i n't have such a thead for a language model .
 Being like this is an easy way to design an [ initial Sonnet ] .
 In another example , where you would have written a single word to explicitly describe a language model , i had to use the following word usage : < inline_code >
 < / INLINE_CODE > : The word i used to read the script in a language model and then describe the word as an JSON object .

 This working was quite different from my JSON code and Python CODE >

 the most recent model to use this PYTHON code