add novel-writing crew

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+crewai-output*/
+models/
+venv/
+
+ollama
+dev.env

novel/.gitignore

@@ -0,0 +1 @@
+__pycache__/

novel/agents.py

@@ -0,0 +1,90 @@
+from crewai import Agent
+from crew.tools import writing_tools, filesystem_tools, editing_tools
+
+SUMMARY_WARNING = (
+  "Pass on the full context to the next agent. Do not summarize the output of your task."
+  "Always share all of the information you have so that future agents can make the best decision possible."
+  "The world depends on it."
+)
+
+character_designer = Agent(
+  role='Character Designer',
+  goal='Design characters for a {genre} novel based on {topic}',
+  verbose=True,
+  memory=True,
+  backstory=(
+    "You are a character designer with a passion for creating complex characters with compelling motives and amazing challenges. "
+    "You are tasked with designing the characters for the {genre} novel, including their appearance, personality, and backstory. "
+  ) + SUMMARY_WARNING,
+  tools=writing_tools,
+  allow_delegation=True
+)
+
+world_researcher = Agent(
+  role='World Developer',
+  goal='Imagine and document a deep and amazing world for a {genre} novel based on {topic}',
+  verbose=True,
+  memory=True,
+  backstory=(
+    "You are a researcher with a passion for developing {setting} with compelling characters,"
+    "realistic motives, and amazing challenges. You are tasked with researching and documenting the world"
+    "and its inhabitants, including the flora and fauna, and the different nations and biomes. "
+  ) + SUMMARY_WARNING,
+  tools=writing_tools,
+  allow_delegation=True
+)
+
+librarian = Agent(
+  role='Librarian',
+  goal='Save the story and world to the filesystem',
+  verbose=True,
+  memory=True,
+  backstory=(
+      "You are a data entry specialist who excels at organizing and saving the content that you are given. "
+      "You save everything you are told to a file and you help your other co-workers read content from files. "
+  ) + SUMMARY_WARNING,
+  tools=filesystem_tools,
+  allow_delegation=False
+)
+
+writer = Agent(
+  role='Writer',
+  goal='Elaborate on the world and characters and write fascinating stories about {topic}',
+  verbose=True,
+  memory=True,
+  backstory=(
+    "You craft engaging narratives and stories that bring the {setting} to light in an accessible manner,"
+    "making the characters come to life and the world feel real. You are a writer with a passion for"
+    "creating {tone} stories. "
+  ) + SUMMARY_WARNING,
+  tools=writing_tools,
+  allow_delegation=True
+)
+
+editor = Agent(
+  role='Editor',
+  goal='Examine chapters of the novel for correct grammar, spelling, and continuity',
+  verbose=True,
+  memory=True,
+  backstory=(
+    "With an eye for detail, you find and fix writing mistakes in {genre} stories. "
+    "You fix any grammatical and spelling errors, then examine the story for continuity,"
+    "making sure all of the details are in agreement and the story contextually makes sense. "
+  ) + SUMMARY_WARNING,
+  tools=editing_tools,
+  allow_delegation=True
+)
+
+publisher = Agent(
+  role='Publisher',
+  goal='Write a {genre} novel about {topic} and save it to a markdown file.',
+  verbose=True,
+  memory=True,
+  backstory=(
+    "You are the publisher for a {genre} novel. You help manage and coordinate your co-workers to write"
+    "chapters of the novel and save them to a file. You then publish the novel when it is complete, by "
+    "saving it to a markdown file. "
+  ) + SUMMARY_WARNING,
+  tools=[*editing_tools, *filesystem_tools],
+  allow_delegation=True
+)

novel/main.py

@@ -0,0 +1,47 @@
+from crewai import Crew, Process
+from crewai.telemetry import Telemetry
+from langchain_openai import ChatOpenAI
+from crew.agents import character_designer, world_researcher, writer, editor, publisher
+from crew.tasks import novel_task, chapter_task, edit_task, world_task, character_task, plot_task
+
+
+# disable spyware, it's broken anyway
+def noop(*args, **kwargs):
+    pass
+
+for attr in dir(Telemetry):
+    if callable(getattr(Telemetry, attr)) and not attr.startswith("__"):
+        setattr(Telemetry, attr, noop)
+
+
+# set up the crew
+crew = Crew(
+    agents=[
+        publisher,
+        editor,
+        writer,
+        world_researcher,
+        character_designer,
+        # librarian,
+    ],
+    tasks=[
+        world_task,
+        character_task,
+        plot_task,
+        chapter_task,
+        edit_task,
+        novel_task,
+    ],
+    manager_llm=ChatOpenAI(temperature=0.05, model="mixtral"),
+    process=Process.hierarchical,
+    verbose=True,
+)
+
+# write a book
+result = crew.kickoff(inputs={
+    'topic': 'a magical porcupine',
+    'setting': 'a rich fantasy world',
+    'genre': 'fantasy',
+    'tone': 'friendly but mysterious',
+})
+print(result)

novel/tasks.py

@@ -0,0 +1,114 @@
+from crewai import Task
+from crew.tools import writing_tools, editing_tools, filesystem_tools
+from crew.agents import character_designer, world_researcher, writer, editor, publisher
+
+class LogTask(Task):
+  def _save_file(self, result):
+    with open(self.output_file, 'a') as f:
+      f.write('\n---\n')
+      f.write(result)
+
+character_task = LogTask(
+  description=(
+    "Design a character for a {genre} novel based on {topic}."
+    "Provide a detailed description of the character's appearance, personality, and backstory."
+  ),
+  expected_output='A comprehensive character description with appearance, motivation, and backstory.',
+  tools=writing_tools,
+  agent=character_designer,
+  async_execution=False,
+  output_file='novel-character.md',
+)
+
+world_task = LogTask(
+  description=(
+    "Research and develop a {setting}."
+    "Provide a rich and detailed description of the world, including the flora and fauna in different biomes,"
+    "the nations that inhabit them, and the challenges faced by their people."
+  ),
+  expected_output='A comprehensive 5 paragraph long report.',
+  tools=writing_tools,
+  agent=world_researcher,
+  async_execution=False,
+  output_file='novel-world.md',
+)
+
+plot_task = LogTask(
+  description=(
+    "Create a plot for a {genre} novel about {topic} set in {setting} with a {tone} tone."
+  ),
+  expected_output='A comprehensive plot outline with at least 5 major plot points.',
+  tools=writing_tools,
+  agent=writer,
+  async_execution=False,
+  output_file='novel-plot.md',
+)
+
+chapter_task = LogTask(
+  description=(
+    "Write a chapter of a {genre} novel. Expand on the existing characters and world,"
+    "focusing on the journeys they embark on and the challenges they face along the way."
+  ),
+  expected_output='A 5 paragraph chapter for a {genre} novel.',
+  tools=editing_tools,
+  agent=writer,
+  async_execution=False,
+  output_file='novel-chapter.md'
+)
+
+edit_task = LogTask(
+  description=(
+    "Edit a chapter of a {genre} novel to ensure correct grammar and spelling and look for continuity errors."
+    "Make sure the story is cohesive, makes sense, and is well-written."
+    "It should be accessible by a reader who is not familiar with the world."
+  ),
+  expected_output='A 5 paragraph chapter for a {genre} novel with correct grammar and spelling.',
+  tools=editing_tools,
+  agent=editor,
+  async_execution=False,
+  output_file='novel-edit.md'
+)
+
+novel_task = LogTask(
+  description=(
+    "Publish a {genre} novel about {topic} set in {setting} with a {tone} tone."
+  ),
+  expected_output='A {genre} novel with 5 chapters and 10,000 words.',
+  tools=editing_tools,
+  agent=publisher,
+  async_execution=False,
+  output_file='novel.md',
+)
+
+save_chapter = LogTask(
+  description=(
+    "Save a completed chapter to a file."
+  ),
+  expected_output='The filename where the chapter was saved.',
+  tools=filesystem_tools,
+  agent=publisher, # librarian,
+  async_execution=False,
+  output_file='novel-save-chapter.md',
+)
+
+save_character = LogTask(
+  description=(
+    "Save a completed character to a file."
+  ),
+  expected_output='The filename where the character was saved.',
+  tools=filesystem_tools,
+  agent=publisher, # librarian,
+  async_execution=False,
+  output_file='novel-save-character.md',
+)
+
+save_world = LogTask(
+  description=(
+    "Save a completed world to a file."
+  ),
+  expected_output='The filename where the world was saved.',
+  tools=filesystem_tools,
+  agent=publisher, # librarian,
+  async_execution=False,
+  output_file='novel-save-world.md',
+)

novel/tools.py

@@ -0,0 +1,85 @@
+import os
+from crewai_tools import tool
+from spellchecker import SpellChecker
+
+spell = SpellChecker()
+ROOT_PATH = "/mnt/optane/ollama/crewai-output"
+
+def safe_name(name):
+    name = name.replace('\\', '')
+    if all([c.isalnum() or c in ('_', '-', '.') for c in name]):
+        return name
+
+    raise ValueError(f"Invalid name: {name}")
+
+
+@tool("spellcheck words")
+def spellcheck_words(text: str) -> list[str]:
+    """Spellcheck a string and return a list of misspelled words."""
+
+    text = text.replace(",", "").replace(".", "") # remove punctuation
+    words = text.split()
+    words = [word for word in words if word == word.lower()]  # remove proper nouns
+
+    return list(spell.unknown(words))
+
+
+@tool("correct spelling")
+def correct_spelling(word: str) -> str:
+    """Get a list of possible correct spellings for a single misspelled word."""
+    return list(spell.candidates(word))
+
+
+@tool("string word length")
+def count_words(text: str) -> int:
+    """Count the total number of words in a string."""
+    return len(text.split())
+
+
+@tool("file word length")
+def count_words_file(filename: str) -> int:
+    """Count the total number of words in a file."""
+    with open(os.path.join(ROOT_PATH, safe_name(filename)), 'r') as f:
+        text = f.read()
+        return len(text.split())
+
+
+@tool("file exists")
+def file_exists(filename: str) -> bool:
+    """Check if a file exists in the output directory."""
+    return os.path.isfile(os.path.join(ROOT_PATH, safe_name(filename)))
+
+
+@tool("write file")
+def write_file(filename: str, content: str) -> None:
+    """Save the content parameter to a file."""
+    path = os.path.join(ROOT_PATH, safe_name(filename))
+    print(f"saving to {path}")
+
+    with open(path, 'a') as f:
+        f.write(content)
+        return content
+
+
+@tool("read file")
+def read_file(filename: str) -> None:
+    """Read some content from a file."""
+    path = os.path.join(ROOT_PATH, safe_name(filename))
+    print(f"reading from {path}")
+
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"File {filename} not found")
+
+    with open(path, 'r') as f:
+        return f.readlines()
+
+
+@tool("list files")
+def list_files() -> list[str]:
+    """List all files in the output directory."""
+    return [f for f in os.listdir(ROOT_PATH) if os.path.isfile(os.path.join(ROOT_PATH, f))]
+
+
+editing_tools = [count_words, spellcheck_words, correct_spelling]
+filesystem_tools = [file_exists, write_file, list_files, read_file, count_words_file]
+writing_tools = []