Chatbot 04: Code organization using separate files and functions

Organizing code is essential to future work on code. It allows for re-use, which increases development velocity. Done correctly it provides ease of initial use, which in turn decreases the amount of time someone else needs to understand your code. There are a plethora of approaches for code organization – and more then one of them are right. How I think about code is not how someone else thinks about code. But, with a decent amount of organization – we can both understand the code and build upon each others successes. It is one thing to develop code in a vacuum. The next level is building in tandem with a team.

Let’s dive right into it. https://github.com/Waryway/chatbot/tree/primary/chatbot-lesson-04-organize is the fourth directory for the chatbot. If you remember from the 3rd lesson – our main.go file was getting a bit lengthy. Nothing terribly egregious, but just enough where we can use it for an example on reorganizing code. Now, bear with me – there is a lot of code listed below – feel free to scroll past it.

Organized Code Files

// main.go
// main function
func main() {
	var history handlers.History
	err := history.Initialize("history.txt").LoadHistory()

	// check for the error.
	if err != nil { // a 'nil' error means no error occurred.
		fmt.Println(err) // print the error
		return           // stop processing the context.
	}

	var echo handlers.EchoBot
	echo.Initialize(history).GreetUser().ChatLoop()
}

// history.go

type History struct {
	Location string
	Current  []string
	file     *os.File
}

func (h *History) Initialize(location string) *History {
	h.Location = location
	return h
}

func (h *History) LoadHistory() error {

	err := h.open()
	if err != nil {
		return err
	}

	defer func() { _ = h.close() }()
	// Slice for storing the History of the world.
	scanner := bufio.NewScanner(h.file) // use the buffer input output package to start a file scanner
	for scanner.Scan() {                // loop through the file scanner
		h.addRecord(scanner.Text() + "\r\n") // add the scanned line to the lines slice.
	}

	return nil
}

func (h *History) Record(record string, isPrintable bool) {
	if isPrintable {
		fmt.Print(record + "\r\n")
	}
	h.addRecord(record + "\r\n")
	h.writeRecord(record + "\r\n")
}

func (h *History) addRecord(record string) {
	h.Current = append(h.Current, record)
}

func (h *History) writeRecord(record string) {
	_ = h.open()
	defer func() { _ = h.close() }()
	_, _ = h.file.WriteString(record)
}

func (h History) SaveAll() {
	for _, record := range h.Current {
		h.writeRecord(record)
	}
}

/**
 * This will perform the three error producing steps to erase the History file.
 * 1. Close the writer
 * 2. Truncate the History file and the local History
 * 3. Re-open the file
 */
func (h *History) EraseHistory() error {
	err := h.close()
	if err == nil {
		err = os.Truncate(h.Location, 0)
	}
	if err == nil {
		h.Current = []string{}
		err = h.open()
	}

	return err
}

func (h History) close() error {
	return h.file.Close()
}

func (h *History) open() error {
	f, err := os.OpenFile(h.Location, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	h.file = f
	// check for the error.
	if err != nil { // a 'nil' error means no error occurred.
		fmt.Println(err) // print the error
		return err       // stop processing the context.
	}

	return nil
}

func (h History) Length() int {
	return len(h.Current)
}

func (h History) Print() {
	// iterate (loop) through the history
	for k, v := range h.Current {
		// print out the Key and the Value
		fmt.Print(k, v)
	}
}

// echobot.go
type EchoBot struct {
	greeting   string
	userPrompt string
	history    History
}

func (e EchoBot) Initialize(history History) EchoBot {
	e.greeting = "Hello, World! \n I am Echo! Please tell me something to say by typing it in, and pressing enter!"
	e.userPrompt = ">"
	e.history = history
	return e
}

func (e EchoBot) GreetUser() EchoBot {
	e.history.Record(e.greeting, true)
	return e
}

func (e EchoBot) ChatLoop() EchoBot {
	// string for storing input
	var input string

	// Stop when we see "bye"
	for input != "bye" {
		e.prompt()
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			input = scanner.Text()
			break
		}

		// Add the new input to the history
		e.history.Record(e.userPrompt+input, false)
		e.history.Record("Echo: "+input, true)
		// Check if the input was the string 'history'
		if input == "history" {
			// We won't add the history to the history, instead we can add how many lines of history exist.
			e.history.Record("<Truncated> "+strconv.Itoa(e.history.Length())+" Lines of history repetition.", false)
			e.history.Print()
		}
	}
	return e
}

func (e EchoBot) prompt() {
	fmt.Print(e.userPrompt)
}

Lesson 3 Code Reference

// main function
func main() {
	f, err := os.OpenFile("history.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)

	// check for the error.
	if err != nil { // a 'nil' error means no error occurred.
		fmt.Println(err) // print the error
		return           // stop processing the context.
	}

	defer func() {
		_ = f.Close()
	}()

	// string for storing input
	var input string

	// Slice for storing the history of the world.
	var history []string
	scanner := bufio.NewScanner(f) // use the buffer input output package to start a file scanner
	for scanner.Scan() {           // loop through the file scanner
		history = append(history, scanner.Text()) // add the scanned line to the lines slice.
	}

	fmt.Println("Hello, World!")
	fmt.Println("I am Echo! Please tell me something to say by typing it in, and pressing enter!")
	_, _ = f.WriteString("Hello, World! \n")
	_, _ = f.WriteString("I am Echo! Please tell me something to say by typing it in, and pressing enter! \n")
	// Stop when we see "bye"
	for input != "bye" {

		fmt.Print("You: ")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			input = scanner.Text()
			break
		}
		_, _ = f.WriteString("You: " + input + "\n")
		// Add the new input to the history
		history = append(history, input)
		fmt.Print("Echo: " + input + "\n")
		_, _ = f.WriteString("Echo: " + input + "\n")
		// Check if the input was the string 'history'
		if input == "history" {
			// We won't add the history to the history, instead we can add how many lines of history exist.
			_, _ = f.WriteString("<Truncated> " + strconv.Itoa(len(history)) + " Lines of history repetition.")
			// iterate (loop) through the history
			for k, v := range history {
				// print out the Key and the Value
				fmt.Println(k, v)
			}
		}
	}
}

Break down of changes

  • Created an independent concept of History
  • Created an independent concept of Echobot
  • Main handles really only handles the initialization, and kickoff of the Echobot

But how did I come to decide those were useful items? Well, looking through the original main.go file, I noticed we were doing a lot of ‘history’ type operations. In noticing this, I considered the concept that, maybe in the future – something else might need to write history. I also considered that idea that history might need to be written to another location. Separating history, and setting it into its own area proved a bit more work then just isolating the same calls from main.


Step 1. Creating a history object

I work a bit through trial and error. I’ve learned in the past, that I like talking to objects. It keeps an object in its own state, it allows me to mock out an object on a whim for testing. It even gives me operations on the object in a simple to reference location. With this in mind, I created the History struct. I then proceeded to add an initialization step. This just makes certain the object knows where it will persist data. It keeps things simple.

Step 2: Adding functionality

Because an object was used – I am able to call the object’s functions and keep the objects state readily available. This will allow me to potentially have different histories, but at the same time, largely avoid having to manage which is what when running methods for the objects.

To figure out the functions for the object, I reference what the main.go originally did. Then, I proceed to create distinct functions for file i/o or history operations. This makes it much easier to find a problem in talking to files, or in actually ‘logical’ operations – such as Print, or Length.

Step 3: Repeating the process somewhere else.

After going through the history object – I quickly came to realize that the main code was still a bit lengthy. (Apologies, I didn’t record the code in this state!). Evaluating what was left in main, and thinking about the goal of this project. I decided the only proper thing to do would be to make an actual echobot struct for handling echo details.

Echobot had some really clear purpose. It needed to listen, it needed to loop, and it absolutely needed to talk to the previously created History object. This quickly led to a lot of dependence on the history object, but, at the same time allowed the concept of a ‘forever listening’ echobot to be present and have its own historical record. This greatly changed the state of the program, without actually really changing anything for the end user.

Summary

Sometimes – you can plan ahead for how you are going to organize pieces of the code base. This will allow you to write tests for you code with ease alongside developing the code, or, in the case of test driven development. It could allow you to design your tests ahead of your code. In this example, we didn’t create any tests. However – with the reorganization of the code – the next step is truly creating tests to make sure things are working as expected. Look out for the next step – Unit Testing your Chatbot

Chatbot 3: Long Term Memory using a file

In the previous post, we covered Golang Slices used for short term memory for the chatbot. As we noted – this only lasts as long as the program is running. Now, let’s see if we can keep information when the chatbot goes offline. That is – long term memory. There are a few different approaches for long term memory. Writing to a file or to a database are the most common. For this example, we’re going to use a file. For future development, most use a database. This allows the database to handle the i/o (input/output) and maintain performance. This is because a database is designed specifically for the use case of storing and retrieving data.

Step 1: Breaking it down

Conceptually, File I/O is actually much more straightforward then previous concepts. The program needs a destination file, a payload, and permission when writing to a file. It needs a source file, destination and permission when reading from a file. It will also need to know how to interpret or parse the data when reading. Thus, it also needs to store the payload in a consistent format. There are a large number of guides in any given programming language for writing to / reading from a file. I’m not going to recommend one currently – and will focus on the actual implementation.

Step 2: File I/O in golang

The first example we have, is creating a file. Here is how to create ‘test.txt’ and avoid a memory leak. A memory leak is when something is reference by the program, but the code flow of the program no longer has context for that reference.

// Get a read/write file reference by creating it if it doesn't exist, or appending to it if it does exist.
file, err := os.OpenFile("test.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) // This provides a reference to the file 'f', or an error if it cannot. 

// check for the error. 
if err != nil { // a 'nil' error means no error occurred. 
  fmt.Println(err) // print the error
  return // stop processing the context.
}

// ##IMPORTANT## Always close the file reference to prevent a memory leak!
defer func() { // defer is an instruction that runs when the 'context' closes.
	_ = file.Close() // close the reference!, the underscore is a way to ignore the value returned by Close()
}()

Reading from an open file reference

// Reading each line in a file into an array
var lines []string // declare the slice to hold the data
scanner := bufio.NewScanner(file) // use the buffer input output package to start a file scanner
for scanner.Scan() { // loop through the file scanner
  lines = append(lines, scanner.Text()) // add the scanned line to the lines slice.
}

Writing to an open file reference

text := "Coming to a file near you" // string to write to file
_, err = fmt.Fprintln(file, text) // write the data to the file
if err != nil { // error handling if file I/O fails.
   fmt.Println(err)
}

\\ or alternatively
linesWritten, err := f.WriteString(text)

Step 3: Date storage format

There are many ways data can be stored within a file. If you’ve messed with files in the past – you’ll know a bit about text .txt files. Maybe you’ve seen markdown .md files. There are many different types of files – and these all serve different purposes. markdown, text, and even .csv files are all Human Readable Formats. If you open any of these in a text editor, you’ll be able to make sense of them quickly. However – they aren’t exactly the best for storing data from a program. For this particular example, we are going to simply use a txt file to place human readable data into the file.

Step 4: Chatbot Enhanced Memory

  1. launch program
  2. check if history exists (create it if it doesn’t exist)
    1. load the history into the history slice
  3. send a friendly greeting and store it to history
  4. send a simple instruction and store it to history
  5. Have the user put something in and store it to history
  6. echo what the user put in and store it to history
  7. Check if the user typed ‘history’
    1. Store the request to history
    2. don’t repeat the history in the history file, just count the number of lines.
    3. Loop through the history slice
    4. print the key and the value
  8. repeat 4-7 until the user says ‘bye’

Step 5: Putting it all together

Now that we’ve covered the details, we can run the program. We’ll find that it isn’t much different from running the program during lesson 2. However – when we say ‘bye’ and start the program again, that’s where the changes are noticeable.

  1. run the program
  2. type hello
  3. type history
  4. type bye
  5. run the program again
  6. type history

Example output

Here is the final example used above. It is also available in github in chatbot, lesson 3

func main() {
	f, err := os.OpenFile("history.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)

	// check for the error.
	if err != nil { // a 'nil' error means no error occurred.
		fmt.Println(err) // print the error
		return           // stop processing the context.
	}

	defer func() {
		_ = f.Close()
	}()

	// string for storing input
	var input string

	// Slice for storing the history of the world.
	var history []string
	scanner := bufio.NewScanner(f) // use the buffer input output package to start a file scanner
	for scanner.Scan() {           // loop through the file scanner
		history = append(history, scanner.Text()) // add the scanned line to the lines slice.
	}

	fmt.Println("Hello, World!")
	fmt.Println("I am Echo! Please tell me something to say by typing it in, and pressing enter!")
	_, _ = f.WriteString("Hello, World! \n")
	_, _ = f.WriteString("I am Echo! Please tell me something to say by typing it in, and pressing enter! \n")
	// Stop when we see "bye"
	for input != "bye" {

		fmt.Print("You: ")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			input = scanner.Text()
			break
		}
		_, _ = f.WriteString("You: " + input + "\n")
		// Add the new input to the history
		history = append(history, input)
		fmt.Print("Echo: " + input + "\n")
		_, _ = f.WriteString("Echo: " + input + "\n")
		// Check if the input was the string 'history'
		if input == "history" {
			// We won't add the history to the history, instead we can add how many lines of history exist.
			_, _ = f.WriteString("<Truncated> " + strconv.Itoa(len(history)) + " Lines of history repetition.")
			// iterate (loop) through the history
			for k, v := range history {
				// print out the Key and the Value
				fmt.Println(k, v)
			}
		}
	}
}

I don’t know about you – but that code is starting to look like my pantry after my kids try finding the peanut butter. In our next lesson – we will start to look at organizing code a little bit better.

Chatbot 2: Improve the chatbot with a memory

Many people can recount their first memory. We don’t really know how we do it, or why it is a first memory. It’s just there. We can also remember things more recently – or – remember information that is imaginary. For this lesson, we will be covering memory in terms of programming. Because a chatbot without a history, is like playing Memory and just choosing random cards each turn. Lesson examples can be found on Github

Step 1: Understanding the Lesson

Before we dig into code – let’s see if we can understand a little more about storing things in program memory. For most languages a developer can write, there are a few different ways to store data. In previous lessons, we’ve covered what a variable is – including a few types of variables. In this lesson, we are diving more deeply into the array type of variable.

One way to understand an array, is to think of series of mailboxes. They all have a number on them so the delivery person can put the mail into the correct slot. When a large neighborhood is built, there are only so many houses – and – there are only so many mailboxes. Depending on how this neighborhood is setup, any new houses will either be an easy addition (or) for a grouped mailbox setup – a new group would eventually need to be created. Arrays are like the grouped mailbox setup.

But what is an array, really? An array is a declared list of keys and values. From the mailbox example, the mailbox number is the key, and the things inside the mailbox are the value. At a more physical layer – arrays are blocks of ram reserved for storing sets of specific types of variables. In different programming languages, there are different mechanics for using arrays. In c++, one needs to program what happens if you try to add more keys to an array then the original requested amount. In golang – the language itself handles this for you using slices – so you don’t have to program the tedious memory management. Most modern languages handle the array management operations in some similar fashion.

Step 2: Using an Array in Golang

Now that we’ve briefly covered the definition of an array, let’s look at a few array operations. The Golang Tour provides an example of arrays here: A Tour of Go (golang.org).

Example from the golang tour with notes:

func main() {
  // creating a new array
  var a [2]string

  // adding to an array using a key
  a[0] = "Hello"
  a[1] = "World"

  // looking up something in an array
  fmt.Println(a[0], a[1]) // Prints Hello World

  // Creating an array with values
  primes := [6]int{2, 3, 5, 7, 11, 13}
  fmt.Println(primes) // Print [2 3 5 7 11 13]

  // removing the item with key 3 from the array
  var slice []int                                // create a 
  slice - notice how it doesn't have a length specified?
  slice = append(slice, 1, 2, 3, 4, 5, 6) // add values 
  to the array
  removalKey := 3                                       // set a key for removal

  // remove an item by adding the first portion (before the key, and the second half after the key together) 
  slice = append(slice[:removalKey], 
  slice[removalKey+1:]...)
  fmt.Println(slice) // Print [1 2 3 5 6]        // notice how the '4' is gone?

  // iterate (loop) through the slice and print the key and the value
  for key, value := range slice {
    fmt.Println(k, v)
  } 
}

Step 3: Chatbot wants a slice

With only a few lines of change, we are going to have chatbot keep a short term memory of the conversation. Here is a breakdown of the logic we’re going to use for chatbot:

  1. launch program
  2. send a friendly greeting
  3. send a simple instruction
  4. Have the user put something in
  5. echo what the user put in
  6. Check if the user typed ‘history’
    1. Loop through the history slice
    2. print the key and the value
  7. repeat 4-6 until the user says ‘bye’

To start, we will initialize a new slice.

// Slice for storing the history of the world.
var history []string

Next, whenever we get new input, we are going to store it in the history

// Add the new input to the history
history = append(history, input)

Finally, chatbot is going to check for a ‘prompt’ that it will use to print the history.

// Check if the input was the string 'history'
if input == "history" {
	// iterate (loop) through the history
	for k, v := range history {
		// print out the Key and the Value
		fmt.Println(k, v)
	}
}

Step 4: Putting it all together

The completed above code can be found in this github repository. There is a main.go file (the code) and a README.md file (the directions).

To get the following output, run the program, and enter the following lines

  1. type test
  2. type cool
  3. type history
Program Output

Well, we’ve now got a chatbot that is still a copycat. It can now recount all the things it has copied. This ‘history’ only lasts while the program is running. In future lessons, we can explore more permanent memory, command logic, and making the code organized.