diff --git a/2023.md b/2023.md index 3a22d96..a380151 100644 --- a/2023.md +++ b/2023.md @@ -156,7 +156,7 @@ Or contact us via Twitter, my handle is [@MichaelCade1](https://twitter.com/Mich ### Engineering for Day 2 Ops -- [] 👷🏻‍♀️ 84 > [](2023/day84.md) +- [✔] 👷🏻‍♀️ 84 > [Writing an API - What is an API?](2023/day84.md) - [] 👷🏻‍♀️ 85 > [](2023/day85.md) - [] 👷🏻‍♀️ 86 > [](2023/day86.md) - [] 👷🏻‍♀️ 87 > [](2023/day87.md) diff --git a/2023/day2-ops-code/README.md b/2023/day2-ops-code/README.md new file mode 100644 index 0000000..d411428 --- /dev/null +++ b/2023/day2-ops-code/README.md @@ -0,0 +1,44 @@ +# Getting started + +This repo expects you to have a working kubernetes cluster already setup and +available with kubectl + +We expect you already have a kubernetes cluster setup and available with kubectl and helm. + +I like using (Civo)[https://www.civo.com/] for this as it is easy to setup and run clusters + +The code is available in this folder to build/push your own images if you wish - there are no instructions for this. + +## Start the Database +```shell +kubectl apply -f database/mysql.yaml +``` + + +## deploy the day1 - sync +```shell +kubectl apply -f synchronous/k8s.yaml +``` + +Check your logs +```shell +kubectl logs deploy/generator + +kubectl logs deploy/requestor +``` + +## deploy nats +helm repo add nats https://nats-io.github.io/k8s/helm/charts/ +helm install my-nats nats/nats + +## deploy day 2 - async +```shell +kubectl apply -f async/k8s.yaml +``` + +Check your logs +```shell +kubectl logs deploy/generator + +kubectl logs deploy/requestor +``` diff --git a/2023/day2-ops-code/database/mysql.yaml b/2023/day2-ops-code/database/mysql.yaml new file mode 100644 index 0000000..3806eaa --- /dev/null +++ b/2023/day2-ops-code/database/mysql.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql +spec: + ports: + - port: 3306 + selector: + app: mysql + clusterIP: None +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql +spec: + selector: + matchLabels: + app: mysql + strategy: + type: Recreate + template: + metadata: + labels: + app: mysql + spec: + containers: + - image: mysql:5.6 + name: mysql + env: + # Use secret in real usage + - name: MYSQL_ROOT_PASSWORD + value: password + - name: MYSQL_DATABASE + value: example + - name: MYSQL_USER + value: example + - name: MYSQL_PASSWORD + value: password + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mysql-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +# https://kubernetes.io/docs/tasks/run-application/run-single-instance-stateful-application/ \ No newline at end of file diff --git a/2023/day2-ops-code/synchronous/generator/Dockerfile b/2023/day2-ops-code/synchronous/generator/Dockerfile new file mode 100644 index 0000000..9e40abf --- /dev/null +++ b/2023/day2-ops-code/synchronous/generator/Dockerfile @@ -0,0 +1,17 @@ +# Set the base image to use +FROM golang:1.17-alpine + +# Set the working directory inside the container +WORKDIR /app + +# Copy the source code into the container +COPY . . + +# Build the Go application +RUN go build -o main . + +# Expose the port that the application will run on +EXPOSE 8080 + +# Define the command that will run when the container starts +CMD ["/app/main"] diff --git a/2023/day2-ops-code/synchronous/generator/go.mod b/2023/day2-ops-code/synchronous/generator/go.mod new file mode 100644 index 0000000..efa37a3 --- /dev/null +++ b/2023/day2-ops-code/synchronous/generator/go.mod @@ -0,0 +1,5 @@ +module main + +go 1.20 + +require github.com/go-sql-driver/mysql v1.7.0 diff --git a/2023/day2-ops-code/synchronous/generator/go.sum b/2023/day2-ops-code/synchronous/generator/go.sum new file mode 100644 index 0000000..7109e4c --- /dev/null +++ b/2023/day2-ops-code/synchronous/generator/go.sum @@ -0,0 +1,2 @@ +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= diff --git a/2023/day2-ops-code/synchronous/generator/main.go b/2023/day2-ops-code/synchronous/generator/main.go new file mode 100644 index 0000000..bcac755 --- /dev/null +++ b/2023/day2-ops-code/synchronous/generator/main.go @@ -0,0 +1,139 @@ +package main + +import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" + "math/rand" + "net/http" + "time" +) + +func generateAndStoreString() (string, error) { + // Connect to the database + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + if err != nil { + return "", err + } + defer db.Close() + + // Generate a random string + // Define a string of characters to use + characters := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + // Generate a random string of length 10 + randomString := make([]byte, 64) + for i := range randomString { + randomString[i] = characters[rand.Intn(len(characters))] + } + + // Insert the random number into the database + _, err = db.Exec("INSERT INTO generator(random_string) VALUES(?)", string(randomString)) + if err != nil { + return "", err + } + + fmt.Printf("Random string %s has been inserted into the database\n", string(randomString)) + return string(randomString), nil +} + +func main() { + // Create a new HTTP server + server := &http.Server{ + Addr: ":8080", + } + + err := createGeneratordb() + if err != nil { + panic(err.Error()) + } + + ticker := time.NewTicker(60 * time.Second) + quit := make(chan struct{}) + go func() { + for { + select { + case <-ticker.C: + checkStringReceived() + case <-quit: + ticker.Stop() + return + } + } + }() + + // Handle requests to /generate + http.HandleFunc("/new", func(w http.ResponseWriter, r *http.Request) { + // Generate a random number + randomString, err := generateAndStoreString() + if err != nil { + http.Error(w, "unable to generate and save random string", http.StatusInternalServerError) + return + } + + print(fmt.Sprintf("random string: %s", randomString)) + w.Write([]byte(randomString)) + }) + + // Start the server + fmt.Println("Listening on port 8080") + err = server.ListenAndServe() + if err != nil { + panic(err.Error()) + } +} + +func createGeneratordb() error { + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + if err != nil { + return err + } + defer db.Close() + + // try to create a table for us + _, err = db.Exec("CREATE TABLE IF NOT EXISTS generator(random_string VARCHAR(100), seen BOOLEAN)") + + return err +} + +func checkStringReceived() { + // get a list of strings from database that dont have the "seen" bool set top true + // loop over them and make a call to the requestor's 'check' endpoint and if we get a 200 then set seen to true + + // Connect to the database + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + if err != nil { + print(err) + } + defer db.Close() + + // Insert the random number into the database + results, err := db.Query("SELECT random_string FROM generator WHERE seen IS NOT true") + if err != nil { + print(err) + } + + // loop over results + for results.Next() { + var randomString string + err = results.Scan(&randomString) + if err != nil { + print(err) + } + + // make a call to the requestor's 'check' endpoint + // if we get a 200 then set seen to true + r, err := http.Get("http://requestor-service:8080/check/" + randomString) + if err != nil { + print(err) + } + if r.StatusCode == 200 { + _, err = db.Exec("UPDATE generator SET seen = true WHERE random_string = ?", randomString) + if err != nil { + print(err) + } + } else { + fmt.Println(fmt.Sprintf("Random string has not been received by the requestor: %s", randomString)) + } + } +} diff --git a/2023/day2-ops-code/synchronous/k8s.yaml b/2023/day2-ops-code/synchronous/k8s.yaml new file mode 100644 index 0000000..73426d1 --- /dev/null +++ b/2023/day2-ops-code/synchronous/k8s.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: requestor +spec: + replicas: 1 + selector: + matchLabels: + app: requestor + template: + metadata: + labels: + app: requestor + spec: + containers: + - name: requestor + image: heyal/requestor:sync + imagePullPolicy: Always + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: requestor-service +spec: + selector: + app: requestor + ports: + - name: http + protocol: TCP + port: 8080 + targetPort: 8080 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: generator +spec: + replicas: 1 + selector: + matchLabels: + app: generator + template: + metadata: + labels: + app: generator + spec: + containers: + - name: generator + image: heyal/generator:sync + imagePullPolicy: Always + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: generator-service +spec: + selector: + app: generator + ports: + - name: http + protocol: TCP + port: 8080 + targetPort: 8080 + type: ClusterIP diff --git a/2023/day2-ops-code/synchronous/requestor/Dockerfile b/2023/day2-ops-code/synchronous/requestor/Dockerfile new file mode 100644 index 0000000..9e40abf --- /dev/null +++ b/2023/day2-ops-code/synchronous/requestor/Dockerfile @@ -0,0 +1,17 @@ +# Set the base image to use +FROM golang:1.17-alpine + +# Set the working directory inside the container +WORKDIR /app + +# Copy the source code into the container +COPY . . + +# Build the Go application +RUN go build -o main . + +# Expose the port that the application will run on +EXPOSE 8080 + +# Define the command that will run when the container starts +CMD ["/app/main"] diff --git a/2023/day2-ops-code/synchronous/requestor/go.mod b/2023/day2-ops-code/synchronous/requestor/go.mod new file mode 100644 index 0000000..efa37a3 --- /dev/null +++ b/2023/day2-ops-code/synchronous/requestor/go.mod @@ -0,0 +1,5 @@ +module main + +go 1.20 + +require github.com/go-sql-driver/mysql v1.7.0 diff --git a/2023/day2-ops-code/synchronous/requestor/go.sum b/2023/day2-ops-code/synchronous/requestor/go.sum new file mode 100644 index 0000000..7109e4c --- /dev/null +++ b/2023/day2-ops-code/synchronous/requestor/go.sum @@ -0,0 +1,2 @@ +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= diff --git a/2023/day2-ops-code/synchronous/requestor/main.go b/2023/day2-ops-code/synchronous/requestor/main.go new file mode 100644 index 0000000..98c1ca3 --- /dev/null +++ b/2023/day2-ops-code/synchronous/requestor/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + _ "github.com/go-sql-driver/mysql" + "io" + "net/http" + "strings" + "time" +) + +func storeString(input string) error { + // Connect to the database + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + defer db.Close() + // Insert the random number into the database + _, err = db.Exec("INSERT INTO requestor(random_string) VALUES(?)", input) + if err != nil { + return err + } + + fmt.Printf("Random string %s has been inserted into the database\n", input) + return nil +} + +func getStringFromDB(input string) error { + // see if the string exists in the db, if so return nil + // if not, return an error + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + defer db.Close() + result, err := db.Query("SELECT * FROM requestor WHERE random_string = ?", input) + if err != nil { + return err + } + + for result.Next() { + var randomString string + err = result.Scan(&randomString) + if err != nil { + return err + } + if randomString == input { + return nil + } + } + + return errors.New("string not found") +} + +func getStringFromGenerator() { + // make a request to the generator + // save sthe string to db + r, err := http.Get("http://generator-service:8080/new") + if err != nil { + fmt.Println(err) + return + } + + body, err := io.ReadAll(r.Body) + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(fmt.Sprintf("body from generator: %s", string(body))) + + storeString(string(body)) + +} + +func main() { + // setup a goroutine loop calling the generator every minute, saving the result in the DB + + ticker := time.NewTicker(60 * time.Second) + quit := make(chan struct{}) + go func() { + for { + select { + case <-ticker.C: + getStringFromGenerator() + case <-quit: + ticker.Stop() + return + } + } + }() + + // Create a new HTTP server + server := &http.Server{ + Addr: ":8080", + } + + err := createRequestordb() + if err != nil { + panic(err.Error()) + } + + // Handle requests to /generate + http.HandleFunc("/check/", func(w http.ResponseWriter, r *http.Request) { + // get the value after check from the url + id := strings.TrimPrefix(r.URL.Path, "/check/") + + // check if it exists in the db + err := getStringFromDB(id) + if err != nil { + http.Error(w, "string not found", http.StatusNotFound) + return + } + + fmt.Fprintf(w, "string found", http.StatusOK) + }) + + // Start the server + fmt.Println("Listening on port 8080") + err = server.ListenAndServe() + if err != nil { + panic(err.Error()) + } +} + +func createRequestordb() error { + db, err := sql.Open("mysql", "root:password@tcp(mysql:3306)/mysql") + if err != nil { + return err + } + defer db.Close() + + // try to create a table for us + _, err = db.Exec("CREATE TABLE IF NOT EXISTS requestor(random_string VARCHAR(100))") + + return err +} diff --git a/2023/day84.md b/2023/day84.md index e69de29..a5d7f93 100644 --- a/2023/day84.md +++ b/2023/day84.md @@ -0,0 +1,58 @@ +# Writing an API - What is an API? + +The acronym API stands for “application programming interface”. What does this really mean though? It’s a way of +controlling an application programmatically. So when you use a website that displays some data to you (like Twitter) +there will be an action taken by the interface to get data or send data to the application (the twitter backend in this +example) - this is done programmatically in the background by code running in the user interface. + +In the example given above we looked at an example of a public API, however the vast majority of APIs are private, one +request to the public twitter API will likely cause a cascade of interactions between programs in the backend. These +could be to save the tweet text into a datastore, to update the number of likes or views a tweet has or to take an image +that has been uploaded and resize it for a better viewing experience. + +We build programs with APIs that other people can call so that we can expose program logic to other developers, teams +and our customers or suppliers. They are a predefined way of sharing information. For example, we can define an API +using [openapi specification](https://swagger.io/resources/open-api/) which is used +for [RESTful](https://en.wikipedia.org/wiki/Representational_state_transfer) API design. This api specification forms a +contract that we can fulfil. For example, If you make an API request to me and pass a specific set of content, such as a +date range, I will respond with a specific set of data. Therefore you can reliably expect to receive data of a certain +type when calling my API. + +We are going to build up an example set of applications that communicate using an API for this section of the learning +journey to illustrate the topics and give you a hands-on look at how things can be done. + +Design: +2 programs that communicate bi-directionally, every minute or so one application will request a random string from the +other, once one is received it will store this number in a database for future use + +The Random Number generator will generate a random string when requested and save this into a database, the application +will then ask the first program for confirmation that it received the string, and store this information against the +string in the database + +The applications will be called: +generator +requestor + +This may sound like a silly example but it allows us to quickly look into the various tasks involved with building, +deploying, monitoring and owning a service that runs in production. There are bi-directional failure modes as each +application needs something from the other to complete successfully and things we can monitor such as API call rates - +We can see if one application stops running. + +We need to now decide what our API Interfaces should look like. We have 2 API calls that will be used to communicate +here. Firstly, the `requestor` will call the `generator` and ask for a string. This is likely going to be an API call +without any additional content other than making a request for a string. Secondly, the `generator` will start to ask +the `requestor` for confirmation that it received and stored the string, in this case we need to pass a parameter to +the `requestor` which will be the string we are interested in knowing about. + +The `generator` will use a URL path of `/new` to serve up random strings +The `requestor` is going to use URL paths to receive string information from the `generator` to check the status, so we +will setup a URL of `/strings/` where is the string of interest. + + +## Building the API +There is a folder on the Github repository under the 2023 section called `day2-ops-code` and we will be using this +folder to store our code for this, and future, section of the learning journey. + +We are using Golang's built in HTTP server to serve up our endpoints and asynchronous goroutines to handle the +checks. Every 60 seconds we will look into the generators database and get all the strings which we dont have +conformation for and then calling the requesters endpoint to check if the string is there. \ No newline at end of file