
Let's review basic concurrency concepts in Java.

## Process vs Thread

Processes are like having multiple programs running at the same time. Each program is a process.

Threads are like a single program running concurrently. A process can have multiple threads.

| Feature          | Process          | Thread           |
|------------------|------------------|------------------|
| Memory           | Isolated         | Shared           |
| Creation Cost    | High             | Low              |
| Communication    | Slow (IPC)       | Fast (shared mem)|
| Crash Effect     | Only this process dies | Can kill the process the thread belongs to |

## Concurrency vs Parellalism

| Feature          | Concurrency                          | Parallelism                          |
|------------------|--------------------------------------|--------------------------------------|
| **Definition**   | Dealing with multiple tasks at once (may or may not execute simultaneously) | Executing multiple tasks *simultaneously* |
| **Execution**    | Tasks make progress together (interleaved) | Tasks run at *exact same time* (on multiple cores) |
| **Hardware**     | Possible on single-core CPU           | Requires multi-core CPU/distributed systems |
| **Goal**         | Manage many tasks efficiently         | Speed up computation                 |
| **Example**      | Handling 1000 web requests on a server (switching between them) | Rendering 4K video by splitting frames across 8 CPU cores |

## Problems in Concurrency

1. Race Condition:
When two threads access shared data at the same time, leading to unpredictable results.
2. Deadlock: 
When two threads block each other forever because each holds a resource the other needs.
3. Starvation:
When a thread never gets CPU time because others always take priority. For example, a low-priority thread never running because high-priority threads keep executing.
4. Livelock:
Threads keep retrying but make no progress (like two people trying to pass each other but always stepping aside the same way).

## Synchronization Basics

### Mutex (Lock)

#### Without a lock

```java
public class MutexLockNotUsed {
	private static int counter = 0;

	public static void main(String[] args) throws InterruptedException {
		Thread t1 = new Thread(() -> incrementCounter(10000));
		Thread t2 = new Thread(() -> incrementCounter(10000));

		t1.start();
		t2.start();

		t1.join();
		t2.join();

		System.out.println("Final counter value without lock: " + counter); // Likely less than 20000
	}

	private static void incrementCounter(int times) {
		for (int i = 0; i < times; i++) {
			counter++; // UNSAFE! No synchronization
		}
	}
}
```

When you write counter++, it looks like a single operation, but the CPU actually performs three distinct steps:

- READ the current value from memory into a register
- INCREMENT the value in the register
- WRITE the new value back to memory

Imagine Thread A and Thread B both executing counter++ when counter starts at 0:

```
Thread A: Reads counter (0)
Thread B: Reads counter (0)  ← Both threads see 0!
Thread A: Writes 1           ← Now memory = 1
Thread B: Writes 1           ← Overwrites with 1 again!
```

Instead of reaching 2, we only get 1 - we've lost an increment.

#### With a lock

```java
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class MutexLock {
	private static int counter = 0;
	private static final Lock lock = new ReentrantLock();

	public static void main(String[] args) throws InterruptedException {
		Thread t1 = new Thread(() -> incrementCounter(10000));
		Thread t2 = new Thread(() -> incrementCounter(10000));

		t1.start();
		t2.start();

		t1.join();
		t2.join();

		System.out.println("Final counter value: " + counter); // Should be 20000
	}

	private static void incrementCounter(int times) {
		for (int i = 0; i < times; i++) {
			lock.lock();
			try {
				counter++;
			} finally {
				lock.unlock();
			}
		}
	}
}
```

The lock ensures thread safety by preventing concurrent access to the shared `counter` variable.

- Mutual Exclusion: When one thread acquires the lock with `lock.lock()`, other threads must wait until the lock is released before they can access the protected code section.
- Atomic Operations: The `try-finally` block ensures the lock is always released, preventing deadlocks while maintaining atomicity of the counter increment operation.
- Memory Visibility: The lock establishes a happens-before relationship, ensuring changes made by one thread are visible to others after the lock is released.

### Semaphore

#### Without a semaphore

```java
public class WithoutSemaphore {
	private static final int MAX_CONCURRENT_DOWNLOADS = 3;
	private static AtomicInteger currentDownloads = new AtomicInteger(0);

	public static void main(String[] args) {
		// Simulate 10 users trying to download
		for (int i = 1; i <= 10; i++) {
			int finalI = i;
			new Thread(() -> downloadFile("User-" + finalI)).start();
		}
	}

	static void downloadFile(String user) {
		int downloads = currentDownloads.incrementAndGet();

		// Check if limit is exceeded (but it's too late!)
		if (downloads > MAX_CONCURRENT_DOWNLOADS) {
			System.err.println(user + " FAILED: Too many downloads (" + downloads + ")");
		} else {
			System.out.println(user + " started downloading... [Active: " + downloads + "]");
		}

		// Simulate download time
		try { Thread.sleep(2000); }
		catch (InterruptedException e) { e.printStackTrace(); }

		currentDownloads.decrementAndGet();
		System.out.println(user + " finished downloading.");
	}
}
```

This code produces a log like below

```
User-3 started downloading... [Active: 2]
User-6 started downloading... [Active: 3]
User-7 started downloading... [Active: 1]
User-10 FAILED: Too many downloads (10)
User-5 FAILED: Too many downloads (8)
User-2 FAILED: Too many downloads (5)
User-9 FAILED: Too many downloads (9)
User-1 FAILED: Too many downloads (7)
User-8 FAILED: Too many downloads (6)
User-4 FAILED: Too many downloads (4)
```

#### With a semaphore

```java
public class WithSemaphore {
	private static final Semaphore semaphore = new Semaphore(3); // Allow 3 at a time

	public static void main(String[] args) {
		for (int i = 1; i <= 10; i++) {
			int finalI = i;
			new Thread(() -> downloadFile("User-" + finalI)).start();
		}
	}

	static void downloadFile(String user) {
		try {
			semaphore.acquire(); // Blocks if >3 downloads
			System.out.println(user + " started downloading... [Active: " + (3 - semaphore.availablePermits()) + "]");

			Thread.sleep(2000); // Simulate download

		} catch (InterruptedException e) {
			e.printStackTrace();
		} finally {
			semaphore.release();
			System.out.println(user + " finished downloading.");
		}
	}
}
```

With a semaphore, we can ensure that only 3 downloads happen a time.

### Producer Consumer

#### Without lock

```java
public class UnsafeProducerConsumer {
    private static final int CAPACITY = 5;
    private static final Queue<Integer> queue = new LinkedList<>();

    public static void main(String[] args) {
        new Thread(UnsafeProducerConsumer::produce).start();
        new Thread(UnsafeProducerConsumer::consume).start();
    }

    private static void produce() {
        int value = 0;
        while (true) {
            // BROKEN: No synchronization
            if (queue.size() < CAPACITY) { // Race condition here
                queue.offer(value);
                System.out.println("Produced: " + value);
                value++;
            }
            // Missing: Condition signaling
        }
    }

    private static void consume() {
        while (true) {
            // BROKEN: No synchronization
            if (!queue.isEmpty()) { // Race condition here
                int value = queue.poll();
                System.out.println("Consumed: " + value);
            }
            // Missing: Condition signaling
        }
    }
}
```

This may cause the producer to produce more than the limit of the queue size.

```
Produced: 0
Produced: 1
Produced: 2
Produced: 3
Produced: 4
Produced: 5 <- greater than the queue size limit of 5.
```

### With lock

```java
public class SafeProducerConsumer {
	private static final int CAPACITY = 5;
	private static final Queue<Integer> queue = new LinkedList<>();
	private static final Lock lock = new ReentrantLock();
	private static final Condition notFull = lock.newCondition();
	private static final Condition notEmpty = lock.newCondition();

	public static void main(String[] args) {
		Thread producer = new Thread(SafeProducerConsumer::produce);
		Thread consumer = new Thread(SafeProducerConsumer::consume);

		producer.start();
		consumer.start();
	}

	private static void produce() {
		int value = 0;
		while (true) {
			lock.lock();
			try {
				while (queue.size() == CAPACITY) {
					System.out.println("Queue full, producer waiting...");
					notFull.await();
				}
				queue.offer(value);
				System.out.println("Produced: " + value);
				value++;
				notEmpty.signal(); // Wake up consumer
				Thread.sleep(500); // Slow down producer
			} catch (InterruptedException e) {
				e.printStackTrace();
			} finally {
				lock.unlock();
			}
		}
	}

	private static void consume() {
		while (true) {
			lock.lock();
			try {
				while (queue.isEmpty()) {
					System.out.println("Queue empty, consumer waiting...");
					notEmpty.await();
				}
				int value = queue.poll();
				System.out.println("Consumed: " + value);
				notFull.signal(); // Wake up producer
				Thread.sleep(1000); // Slow down consumer
			} catch (InterruptedException e) {
				e.printStackTrace();
			} finally {
				lock.unlock();
			}
		}
	}
}
```

The lock ensures that the producer doesn't enqueue greater than the limit of the queue.

```
Produced: 0
Produced: 1
Produced: 2
Produced: 3
Produced: 4
Queue full, producer waiting...
Consumed: 0
```

Basic Concurrency in Java


Refer to previous two posts to deploy a Rails app to Azure.

- [Deploy Rails 7.2.1 + Sqlite3 App to Azure Using Kamal](https://blog.jasonkim.ca/post/deploy-rails-721-sqlite3-app-to-azure-using-kamal)
- [Deploy Rails 7.2.1 + Managed Postgresql App to Azure Using Kamal 2.1.1](https://blog.jasonkim.ca/post/deploy-rails-721-managed-postgresql-app-to-azure-using-kamal-211)

This post is made for:

- Ruby: 3.3.5
- Rails: 8.0.0
- Kamal: 2.3.0 

# 1. Load Balancer Setup

1. Sign into Azure
2. Search for Load Balancers and select it.
3. Select the load balancer for the VM running the Rails app.
4. Select Settings > Inbound NAT rules link in the left sidebar.
5. Click +Add
6. Create a new NAT inbound rule like below
    - ![](/images/2024/12/3/lb-nat-inbound.png)

# 2. Rails App

Setup the Rails like like the following.

## 2.1 `config/environments/production.rb`

```ruby
config.force_ssl = true
```

## 2.2 `config/deploy.yml`

```yaml
...
proxy:
  ssl: true
  host: yourdomain.com
...
```

## 2.3 Deploy and Check

1. Run `kamal deploy`
2. Check `yourdomain.com`

Enable HTTPS for a Rails app with Kamal to Azure


Let's setup a Rails app that uses Postgresql as the DB and deploy it to Azure using Kamal. Note that the Postgresql in production environment will be using the managed Postgresql instance in Azure.

# 1. Settings up Rails App

## 1.1 Create a new Rails app

`$ rails new rails-pg-kamal-deployed-to-azure --minimal --database=postgresql`

## 1.2 Add a scaffold

- `$ cd rails-pg-kamal-deployed-to-azure`
- `$ rails g scaffold post title:string content:text`

## 1.3 Locally set up Postgresql for Testing

- Create `docker-compose.development.yml`

```yml
version: '3.8'
services:
  db:
    image: postgres:16.2-alpine
    container_name: rails-pg-kamal-deployed-to-azure-development-db
    restart: always
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
      - POSTGRES_DB=rails_pg_kamal_deployed_to_azure_development
    ports:
      - '5442:5432'
    volumes:
      - db:/var/lib/postgresql/data
volumes:
  db:
    driver: local
```

- Note the username, password, db name, and the port above

- `config/database.yml`
```yaml
development:
  <<: *default
  database: rails_pg_kamal_deployed_to_azure_development
  port: 5442
  host: localhost
  username: postgres
  password: postgres
```

- Start PG DB locally
  - `$ docker-compose -f docker-compose.development.yml up --build`

## 1.4 Run migration

- You can omit running `$ rails db:create` because database will be created by starting docker-compose
- Run `$ rails db:migrate` 

## 1.5 Add Routes

- `routes.rb`

```ruby
root "posts#index"
```

## 1.6 Start server locally

- `$ ./bin/dev`
- Go to localhost:3000

# 2. Azure

## 2.1 Everything except Postgresql

Please follow [the instructions here](https://blog.jasonkim.ca/post/deploy-rails-721-sqlite3-app-to-azure-using-kamal).

You may want to skip "2.2 Create Container Registry", if you already have a container registry to use.

## 2.2 Postgresql

### 2.2.1 Setting up Basics

1. Go ["Create a resource" page](https://portal.azure.com/#view/Microsoft_Azure_Marketplace/PlusNew.ReactView/package/hub/additionalConfig~/%7B%7D/selectedMenuItemId/undefined/createLanding/undefined)
2. In the left hand side bar, select "Databases".
3. Click "Create" for "Azure Database for PostgreSQL"
4. Select the Subscription and Resource Group
5. Under Server Details, provide info like below.
  - ![](/images/2024/10/azure-pg-server-details.png)
6. Make sure to Click "Configure server" and size down the DB server appropriately.
  - ![](/images/2024/10/azure-pg-server-config.png)
7. Provide info to setup "Authentication" for the db.
  - ![](/images/2024/10/azure-pg-auth.png)
8. Click "Review + create".



PG
- pgadminops
- 8132a28d96bc@

### 2.2.2 Networking

- Setup the Networking setup like below
- ![](/images/2024/10/azure-pg-server-network.png)
- Click "Create".

### 2.2.3 Dealing with `extension "plpgsql" is not allow-listed for "azure_pg_admin" users in Azure Database for PostgreSQL` error.

When you use Azure Database for PostgreSQL, you will see the following error.

```
 ERROR 2024-10-16T06:56:56.367508062Z bin/rails aborted!
2024-10-16T06:56:56.367929864Z ActiveRecord::StatementInvalid: PG::FeatureNotSupported: ERROR:  extension "plpgsql" is not allow-listed for "azure_pg_admin" users in Azure Database for PostgreSQL (ActiveRecord::StatementInvalid)
2024-10-16T06:56:56.367975864Z HINT:  to learn how to allow an extension or see the list of allowed extensions, please refer to https://go.microsoft.com/fwlink/?linkid=2281561
2024-10-16T06:56:56.368497566Z /rails/db/schema.rb:15:in `block in <top (required)>'
2024-10-16T06:56:56.368509766Z /rails/db/schema.rb:13:in `<top (required)>'
2024-10-16T06:56:56.368641767Z
2024-10-16T06:56:56.368669567Z Caused by:
2024-10-16T06:56:56.368799368Z PG::FeatureNotSupported: ERROR:  extension "plpgsql" is not allow-listed for "azure_pg_admin" users in Azure Database for PostgreSQL (PG::FeatureNotSupported)
2024-10-16T06:56:56.368809368Z HINT:  to learn how to allow an extension or see the list of allowed extensions, please refer to https://go.microsoft.com/fwlink/?linkid=2281561
2024-10-16T06:56:56.368998368Z /rails/db/schema.rb:15:in `block in <top (required)>'
2024-10-16T06:56:56.369025368Z /rails/db/schema.rb:13:in `<top (required)>'
2024-10-16T06:56:56.369153969Z Tasks: TOP => db:prepare
2024-10-16T06:56:56.369269569Z (See full trace by running task with --trace)
2024-10-16T06:57:00.413100892Z bin/rails aborted!
```

1. Go to `Azure Database for PostgreSQL`
2. Select the PG database
3. Click Settings > Server Parameters
4. Search for `azure.extensions`.
5. Click `Value` column
6. Check `PLPGSQL`.
  - ![](/images/2024/10/pg-azure-exte.png)

See [link](https://stackoverflow.com/a/74002100/536890) for more info.

# 3. Setting up Kamal 2 for Rails

## 3.1. Follow these steps here

Follow Step 3 in the following [link](https://blog.jasonkim.ca/post/deploy-rails-721-sqlite3-app-to-azure-using-kamal), but you can skip Step 3.1.

## 3.2. Update `.kamal/secrets`

- Add secrets in `.kamal/secrets`.

```
RAILS_MASTER_KEY=$(cat config/master.key)
KAMAL_REGISTRY_PASSWORD=...
POSTGRES_PASSWORD=...
```

## 3.3. Update `config/database.yml`

```yml
production:
  <<: *default
  database: rails_pg_kamal_deployed_to_azure_production
  username: pgadminops
  password: <%= ENV["POSTGRES_PASSWORD"] %>
  host: <%= ENV["DB_HOST"] %>
```

## 3.4. Update `config/deploy.yml`

- Comment out `proxy.host`.

Example:

```yml
<% require "dotenv"; Dotenv.load(".env") %>

# Name of your application. Used to uniquely configure containers.
service: rails-pg-kamal-deployed-to-azure

# Name of the container image.
image: serv/rails-pg-kamal-deployed-to-azure

# Deploy to these servers.
servers:
  web:
    hosts:
      - x.y.z.a  
  # job:
  #   hosts:
  #     - 192.168.0.1
  #   cmd: bin/jobs

# Enable SSL auto certification via Let's Encrypt (and allow for multiple apps on one server).
# Set ssl: false if using something like Cloudflare to terminate SSL (but keep host!).
proxy:
  ssl: false
  # host: 52.146.91.48
  # kamal-proxy connects to your container over port 80, use `app_port` to specify a different port.
  app_port: 3000
  healthcheck:
    path: /up

# Credentials for your image host.
registry:
  # Specify the registry server, if you're not using Docker Hub
  # server: registry.digitalocean.com / ghcr.io / ...
  server: xyz.azurecr.io
  username: xyz

  # Always use an access token rather than real password (pulled from .kamal/secrets).
  password:
    - KAMAL_REGISTRY_PASSWORD

# Configure builder setup.
builder:
  # context: .
  arch: amd64
# Inject ENV variables into containers (secrets come from .kamal/secrets).
#
env:
  clear:
    DB_HOST: <%= ENV["DB_HOST"] %>
  secret:
    - RAILS_MASTER_KEY
    - KAMAL_REGISTRY_PASSWORD
    - POSTGRES_PASSWORD

# Aliases are triggered with "bin/kamal <alias>". You can overwrite arguments on invocation:
# "bin/kamal logs -r job" will tail logs from the first server in the job section.
#
# aliases:
#   shell: app exec --interactive --reuse "bash"

# Use a different ssh user than root
ssh:
  user: azureuser
```

## 3.5 Update `config/environments/production.rb`

Comment out `force_ssl`.

```ruby
# config.force_ssl = true
```

## 3.6. Update `config/routes.rb`

```ruby
root "posts#index"
```



Deploy Rails 7.2.1 + Managed Postgresql App to Azure Using Kamal 2.1.1


[Kamal](https://kamal-deploy.org/) is a deployment tool for 
deploying Rails apps to cloud environments.
And Azure is a fantastic cloud service provider (I work for Microsoft) to
deploy Rails apps.

Let's try to deploy a test Rails app with a Sqlite3 DB to Azure.

Although this guide will go into details about setting up various systems in
Azure, the guide assumes that you have Azure account and billing setup already.
The guide also assumes some preliminary understanding of Rails.

Here are the exact versions of tools I am using as of April 7, 2024.

- Ruby: 3.3.0
- Rails: 7.2.1
- Kamal: 1.3.1
- Sqlite: 3.43.2

# 1. Settings up Rails App

## 1.1 Create a new Rails app

`$ rails new rails-kamal-deployed-to-azure --minimal`

## 1.2 Add a scaffold

- `$ cd rails-kamal-deployed-to-azure`
- `$ rails g scaffold post title:string content:text`

## 1.3 Run migration

- You can omit `rails db:create` because database will be created by starting docker-compose
- Run `$ rails db:migrate` 

## 1.4 Update routes

In `routes.rb`,

```ruby
root "posts#index"
```

## 1.5 Start the server to test locally

- `$ ./bin/rails s`
- Check that the server is locally running.
- Go to [localhost:3000](http://localhost:3000)
- You should see something like this.
    - ![](/images/2024/9/16/16-local-rails.png)

# 2. Azure

You should have a subscription and a resource group for Azure before you proceed.

## 2.1 Create virtual machines

- Go to https://portal.azure.com/#create/Microsoft.VirtualMachine-ARM
- Select a subscription and a resource group.
- Use a name like `rails-kamal-deployed-to-azure-1`
- Choose `SSH public key` for `Authentication type > Administrator account`.
- Add `Key pair name`. Use a name like `rails-kamal-deployed-to-azure-1_key`. 
  - ![](/images/2024/9/16/16-ssh.png)
- Enable ports 22, 80, 443
  - ![](/images/2024/9/16/16-inbound-ports.png)
- Click "Review + create"
- Download the key when prompted and keep it handy.
  - ![](/images/2024/9/16/16-download-key.png)

### 2.1.1. Update SSH config

- Inside `~/.ssh/config`
- Add
    ```
    Host 74.235.199.96
      IdentityFile ./rails-kamal-deployed-to-azure-1_key.pem
      User azureuser
    ```
- `$ ssh-add ~/.ssh/rails-kamal-deployed-to-azure-1_key.pem`
  - You may get 
    ```
    $ ssh-add ~/.ssh/rails-kamal-deployed-to-azure-1_key.pem
    @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    @         WARNING: UNPROTECTED PRIVATE KEY FILE!          @
    @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    Permissions 0644 for '/Users/jasonkim/.ssh/rails-kamal-deployed-to-azure-1_key.pem' are too open.
    It is required that your private key files are NOT accessible by others.
    This private key will be ignored.
    ```
    - `$ chmod 600 ~/.ssh/rails-kamal-deployed-to-azure-1_key.pem`
    - Now you can add it.
      ```
      $ ssh-add ~/.ssh/rails-kamal-deployed-to-azure-1_key.pem
      Identity added: /Users/jasonkim/.ssh/rails-kamal-deployed-to-azure-1_key.pem (/Users/jasonkim/.ssh/rails-kamal-deployed-to-azure-1_key.pem)
      ```
    - check that ssh-ing into the VM works
    - Make sure the VM is turned on.
- SSH into the box
  - `$ sudo usermod -aG docker $USER`

## 2.2 Create Container Registry

- Go to https://portal.azure.com/#browse/Microsoft.ContainerRegistry%2Fregistries
- Create a Container Registry
  - ![](/images/2024/9/16/16-registry.png)
- Once the registry is created, go to the registry resource
- Go to Settings > Access Keys section
- Check "Admin user"
- Copy the passwords

## 2.3 Create a load balancer

- Go to https://portal.azure.com/#create/Microsoft.LoadBalancer-ARM
- Under Basics, setup the load balancer like this
  - ![](/images/2024/9/16/lb.png)
- Click Next: Frontend
- Click "Add a fronted IP Configuration"
- You'll need to create a public IP Address.
- Click "Create New" under Public IP Address
- Create one like this
  - ![](/images/2024/9/16/public-ip.png)
- Click "save" to add frontend IP configuration
- Click "Review + create"
- Click "create"

## 2.4 Setup VM to use the LB

- Go to the VM page
- Click on the VM that should be behind the load balancer
- Click Network > Load balancing 
- Click Add load balancing
- Click Add existing
- Setup load balancing like the following
  - ![](/images/2024/9/16/lb-setup-vm.png)
- Click Save
- This should take upto a minute

## 2.5 Add NAT rule

- Go select the load balancer
- Go to Settings > Inbound NAT Rules
- Add
- Setup the rule like below
  - ![](/images/2024/9/16/natrule.png)

# 3. Kamal Setup

Before you begin, make sure **Docker is locally running** from the machine that you are deploying from.

Follow the instructions found [here](https://kamal-deploy.org/docs/installation/).

- `$ gem install kamal`
- inside your app directory, run `$ kamal init`

## 3.1 Rails Secret

- Run `$ bin/rails credentials:edit`
- Copy the value
- Paste to `.env` file for `RAILS_MASTER_KEY` environment variable.

If you skip this, you will likely face an error like below:

```
 ERROR bin/rails aborted!
ArgumentError: key must be 16 bytes (ArgumentError)

        cipher.key = @secret
                     ^^^^^^^
/rails/config/environment.rb:5:in `<main>'
Tasks: TOP => db:prepare => db:load_config => environment
(See full trace by running task with --trace)
bin/rails aborted!
ArgumentError: key must be 16 bytes (ArgumentError)

        cipher.key = @secret
                     ^^^^^^^
/rails/config/environment.rb:5:in `<main>'
```
    
## 3.1 `config/deploy.yml`

```yml
# Name of your application. Used to uniquely configure containers.
service: rails-kamal-deployed-to-azure

# Name of the container image.
image: serv/rails-kamal-deployed-to-azure

# Deploy to these servers.
servers:
  - 74.235.199.96

# Credentials for your image host.
registry:
  # Specify the registry server, if you're not using Docker Hub
  server: railskamaldeployedtoazure.azurecr.io
  username: railskamaldeployedtoazure

  # Always use an access token rather than real password when possible.
  password:
    - KAMAL_REGISTRY_PASSWORD
# Inject ENV variables into containers (secrets come from .env).
# Remember to run `kamal env push` after making changes!
env:
  #   clear:
  #     DB_HOST: 192.168.0.2
  secret:
    - RAILS_MASTER_KEY
# Use a different ssh user than root
ssh:
  user: azureuser

# To use Sqlite
builder:
  context: .
volumes:
  - 'storage:/rails/storage'
```

## 3.2 `dotenv kamal setup`

- You may get this error: `permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock` when you run `dotenv kamal setup` for the first time.
- For this run `sudo usermod -aG docker ${USER}` [[1]](https://github.com/basecamp/kamal/issues/405#issuecomment-1652527208)
- If the command runs successfully, you should see

```
  INFO [f489cd69] Finished in 0.326 seconds with exit status 0 (successful).
  Finished all in 87.1 seconds
Releasing the deploy lock...
  Finished all in 91.9 seconds
```

- Try going to the IP address of the load balancer.
  - ![](/images/2024/9/16/lbloading.png)

- Also you may or may not need `dotenv` in front of running `kamal setup`.
  - There were changes to how Kamal 2 handles secret environment variables, and to force load the secret variables `dotenv` is needed.
  - [More on this](https://github.com/basecamp/kamal/discussions/977)




Deploy Rails 7.2.1 + Sqlite3 App to Azure Using Kamal


As of Sept 1, 2024, [Microsoft Visual Studio for Mac](https://visualstudio.microsoft.com/vs/mac/) is retired.

![](/images/2024/9/1/vs.png)

This doesn't mean that you can't use Visual Studio on Mac. 
However, Microsoft will not actively develop other than patching 
for security and reliability. [Read more on this.](https://learn.microsoft.com/en-us/visualstudio/releases/2022/what-happened-to-vs-for-mac?view=vsmac-2022)

Thankfully, you can still use VSCode on Mac OSX to develop 
ASP.NET projects.

1. Install prerequisites
    - VSCode
    - [C# Dev Kit extension](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.csdevkit)
    - [Setup .NET development environment](https://blog.jasonkim.ca/post/setup-a-net-project-on-mac-osx)
2. After C# Dev Kit is installed in VSCode, open it in VSCode.
    - You can do this by opening "Extensions" in the left side.
    - Search for C# Dev Kit and open it.
    - Check that the extension is installed.
3. After the extension is installed, press Command + Shift + P.
4. Search for "Welcome: Open Walkthrough".
    - ![](/images/2024/9/1/walkthrough.png)
5. Select "Get Started With C# Dev Kit"
    - ![](/images/2024/9/1/getstarted.png)
6. Under "Create your dotnet project", click "Choose Project Template".
    - ![](/images/2024/9/1/createdotnet.png)
7. Click "New Project"
    - ![](/images/2024/9/1/newproject.png)
8. Now follow the guided prompts, to create the ASP.NET project you want to create.


Starting ASP.NET project in Mac OSX using VSCode


We will create a new .NET project on Mac OSX using VSCode.

If you want to skip ahead and just look at the final code setup, check out this [github repo](https://github.com/serv/csharp-project-setup).

# Setup .NET Development environment

1. Follow instructions on this [link](https://dev.to/rusydy/setting-up-net-on-macos-a-step-by-step-guide-14db).

## Updating to the latest .NET environment 

There's a chance that you had prepped .NET development environment years ago. In such case, we want to update to the latest .NET development environment.

1. Update Homebrew
    - `$ brew update`
2. Install .NET using Homebrew
    - `$ brew install --cask dotnet-sdk`
3. If you already had installed it before but want to get the latest,
    - `$ brew upgrade dotnet-sdk`
4. Verify
    - `$ dotnet --version`
    - Should 8.0.401 as of Aug 31, 2024

# Setup a .NET project

- I am following the [instruction here](https://learn.microsoft.com/en-us/dotnet/core/testing/unit-testing-with-dotnet-test).
- If you want to follow the same instructions with my concise commentary, follow instructions below.

1. Create a .NET project (aka "Solution")
    - `$ dotnet new sln -o project-with-tests`
2. `cd project-with-tests`
3. Creates a new class library project in the SuperMassive folder.
    - `dotnet new classlib -o SuperMassive`
4. Rename Class1.cs to SuperMassive.cs.
5. Inside the project, have the following content.
  ```csharp
  namespace SuperMassiveNS;

  public class SuperMassive
  {

  }
  ```
  - Notice that the namespace is `SuperMassiveNS` in order to avoid collision with the class name.
6. Build
    - `$ dotnet build`
7. Add class library to the solution
    - `$ dotnet sln add ./SuperMassive/SuperMassive.csproj`
8. Create tests
    - `$ dotnet new xunit -o SuperMassive.Tests`
9. Add the tests to the solution
    - `$ dotnet sln add ./SuperMassive.Tests/SuperMassive.Tests.csproj`
10. Add SuperMassive as a dependency to SuperMassive.Tests
    - `$ dotnet add ./SuperMassive.Tests/SuperMassive.Tests.csproj reference ./SuperMassive/SuperMassive.csproj`  
11. Create a test to check that the tests can reference the source class.
    - Rename the test file name
      - SuperMassiveTests.cs
    - Update the content of the test
      ```csharp
      namespace SuperMassive.Tests;

      public class SuperMassiveTests
      {
          [Fact]
          public void ReferenceSourceTest()
          {
              var superMassive = new SuperMassiveNS.SuperMassive();
          }
      }
      ```
12. Run the tests!
    - `$ dotnet test`
    - If you want to run tests with logging
      - `$ dotnet test --logger "console;verbosity=detailed"`

# Adding more source files and tests

1. Go into the source project directory.
    - `$ cd SuperMassive`
2. Add a new class.
    - `$ dotnet new class -n Adder`
3. Update the class
  ```csharp
  namespace SuperMassive;

  public class Adder
  {
    public int add(int i, int j) {
      return i + j;
    }
  }
  ```
4. Go into the test project directory.
5. Add a new test.
    - `$ dotnet new class -n AdderTests`
6. Update the test class
  ```csharp
  namespace SuperMassive.Tests;

  public class AdderTests
  {
    [Fact]
    public void AddTest()
    {
      var adder = new SuperMassive.Adder();
      int result = adder.add(1, 1);
      Assert.Equal(2, result);
    }
  }
  ```
    - Make sure to add `[Fact]`
      - Attribute that is applied to a method to indicate that it is a fact that should be run by the test runner.
7. Run tests
    - `$ dotnet test`
    - Now you should see 2 tests running
      - `Passed!  - Failed:     0, Passed:     2, Skipped:     0, Total:     2, Duration: 1 ms - SuperMassive.Tests.dll (net8.0)`



Setup a .NET project on Mac OSX


Let's setup an Express app using Typescript.

# 1. Install TSC, TSLint, and type declarations for NodeJS

- `npm install --save-dev typescript tslint @types/node @tsconfig/node16`
- Let's install express as well
  - `npm install --save express`

# 2. Write tsconfig.json

```json
{
  "extends": "@tsconfig/node16/tsconfig.json",
  "compilerOptions": {
    "outDir": "dist"
  },
  "include": ["index.ts"]
}
```

# 3. Add npm run commands

We want to be able to run `npm run build` to compile typescript file into a javascript file. 

Add `"build": "./node_modules/.bin/tsc"` under `"scripts"` key.

We also want to run `npm run start` to run the express app.

Add `"start": "node ./dist"` under `"scripts"` key.

The `package.json` file should look like this.

```json
{
  "name": "typescript-nodejs",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "build": "./node_modules/.bin/tsc",
    "start": "node ./dist"
  },
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "@tsconfig/node16": "^1.0.3",
    "@types/node": "^17.0.42",
    "tslint": "^6.1.3",
    "typescript": "^4.7.3"
  },
  "dependencies": {
    "express": "^4.18.1"
  }
}
```

# 4. Add index.ts with express.js

Add the following. THe code is from [the express website](https://expressjs.com/en/starter/hello-world.html).

```typescript
const express = require('express');
const app = express();
const port = 3000;

app.get('/', (req, res) => {
  res.send('Hello World!');
});

app.listen(port, () => {
  console.log(`Example app listening on port ${port}`);
});
```

# 5. Compile and run

- Compile: `npm run build`
- Start: `npm run start`

You should see `Example app listening on port 3000` printed in the console.

You can see the hello world when you go to [localhost:3000](http://localhost:3000/).

You can find the code [here](https://github.com/serv/blog-tutorials/tree/master/typescript-nodejs).


Setting Up Typescript For Node.js in 2022


The year 2022 marks the 10th year I've worked as a software engineer professionally.
There are lots of good memories, but I was noticing that I am starting to forget about the things I've done and learned.
This post is to archive the experience I had as a software engineer for the last 10 years or so.

## Beginning

I learned about Ruby on Rails in 2010 and read [Ruby on Rails Tutorial](https://www.railstutorial.org/book) by Michael Hartl.
I became familiar with the basics of web app development. I started to have hope that maybe I can code a web app on my own.

## Vidyard (Sep 2011 - Dec 2011)

This was my first programming coop job. When I think back, I was really lucky to get this job.
Vidyard is now a pretty big company now in Waterloo, Canada, but at the time I joined, 
there were less than 10 people in the company. 
It was a small company, but the spirit was high. It was so fun working there. The founders cared greatly about everyone they hired.
They made me feel welcome and be part of the company, even though I was just a coop who will work they just for 4 months.
I saw the leaders of the company becoming important focal points in the Waterloo tech scene. 
They may be young and still inexperienced at the time, but they stood out among many founders coming out of Waterloo at the time. 

I learned a lot working just 4 months there. I learned to use javascript and jQuery more. I still was not good at using either, but
exposure to it was important. I also shipped a feature that was sending out emails using Mailgun API. 

Working at Vidyard helped me decide that I want to become a software engineer. This was the most important realization I had 
from working here.

## Rhapsody / Napster (July 2012 - March 2018)

After I finished school around May 2012, I was busy applying to different entry level jobs all over US and Canada.
After failing many different interviews with different companies, I landed a job with Rhapsody, Seattle-based music streaming
company.

I worked on many different projects here. When I first joined the company, I built a FAQ / support page for the company.
Unfortuatenly, it didn't go to production. I then started working on the web streaming application built on Ruby on Rails, jQuery and Flash.
This was early example of a single page web app. When I think back, it's actually amazing that
such complex single page web app could be built with jQuery alone.

After this, I had a chance to work on a brand new project of my own. Napster was preparing international launch in several countries
in Europe and South America. I was asked to create a content managemeng system that can handle multiple regions and languages. 
I built the solution in Ruby on Rails and MySQL on the backend.

I worked on Sountracking application and the webclient to enable social sign in featrues.

I worked on the API Gateway which was built as the service our web and mobile clients was using.
This was the first Node.js application I worked on.

Then I lead the development of the web application for Napster. I didn't realize it as much at the time
but now when I think back, it was another lucky opportunity for me. I was leading the web application for
an app used by 2 million users at the time. It's unbelievable.

The most important thing I did was creating the new audio playback engine for Napster's webclient.
I deprecated the decade old Flash audio playback engine, and replacing it
with the HTML5 Audio player that is DRM compliant with all modern web browsers.

I created the new audio player using [Video.js](https://github.com/videojs/video.js).
The player was bundled with an newly emerging (at the time) builder called 
[Webpack](https://webpack.js.org/).
The hard problem was supporting [EME](https://web.dev/media-eme/) from all the major browsers.
Everything from development, testing and validating was an extreme undertaking and
whenever issues happened, reproducing the issues were challenging.
To make matters even more complicated, the browsers support different streaming protocols (DASH, HLS)
and DRMs (Widevine, Playready, Apple Fairplay).
This accomplishment means a lot to me because when I first joined Napster in 2012, 
the engineers in the web team were discussing ways to deprecate Flash player and move over to HTML5 
player. And in the end, I was able to do that.

At Napster, I learned to become independant and solve difficult technical challenges on my own.

## Coupang (March 2018 - Present)

I joined Coupang's advertising team as the number 2 engineer. The first thing I needed to do
was working on a data pipeline that provided the data foundation for other ad services
to be built. I described my experience building the data pipeline here as a [blog post](https://blog.jasonkim.ca/post/big-data-engineering-with-nodejs). I also gave a 
[talk about it in Seattle.js meetup](https://www.youtube.com/watch?v=HBHStmv4Y8g).

This was one of the toughest onboarding I've had at a new company or a new team ever. There was no engineering 
team to speak of. It was purely myself alone who needed to figure everything out. I understand that we
were in a startup mode, but in retrospect, it's still not a good excuse for not having any support
structure at all for any onboarding. This experience helped me learn how important it is to have a supportive
team and culture to help onboard engineers are.

Briefly, I'll state important components of a successful onboarding system.

- Documentation of systems and runbooks for them
  - Documentations are the most effective ways to scale the onboarding process. Use it.
- Overview of the business
  - Help engineers understand the goal of the overall business
- Mentor / buddy to sync up daily or a couple of times a week
  - Every blockage that seems trivial which may take hours to resolve for a new engineer can be solved in minutes by someone who's already faimiliar with the system

After this I worked on the Reporting UI for advertisers. This was built using Next.js, MySQL and Redis.
The Reporting UI is the window which the advertisers judge the entire Coupang ads offering, so it is important that the app works well.

I am now working on Settlements app. The Settlements app determines how much advertisers need to be 
charged monthly. The app is originally created by another engineer, but I took it over as a tech lead
later. The app is written in Java. 

I am currently rewriting the app using Spark instead of Sprign Batch. The goal behind the redesign is to simplify the
application by modularizing the calculation steps more and making the coupling between different modules looser. 
This allows for easier to understand application that'll be easier to modify over time.

At Coupang, I learned to prioritize work based on business importance.

10 Years As a Software Engineer


I heard great things about [Tailwind.css](https://tailwindcss.com/) and I've wanted to
try it. I also wanted to redesign my blog and stop using
[Octopress](http://octopress.org/).
This was a good opportunity to do both by recreating my blog
using Tailwind.css and move away from using Octopress.
I wanted to make sure I understood how different internal
parts were working together to produce the static pages.
I chose [Next.js](https://nextjs.org/) as it is a popular and well maintained tool
that is capable of turning React components into static
pages.

My blog has two kinds of pages. I have an index page that fetches
all the blog posts and show the list of posts. Another page I
have is a blog post page that shows the content of the blog post.

The index page is created by getting a list of files in the
`_posts` directory where I store all the blog posts in
markdown format. I use [slug](https://www.npmjs.com/package/slug) NPM package to create a
url path for blog posts. I also have to parse the date of the
posts and format it.

The blog post page needs the content of the blog post.
In order to do that, I need to search for the blog post file
using the slug from the url path. Once I get the file,
I need to convert the markdown content to HTML using [remark](https://www.npmjs.com/package/remark).

Now that the data and rendering of the posts are ready, I used
Tailwind.css to style the website. Tailwind.css's utility class
first approach to style is wonderful to use. I haven't written
a single line of my own style. Instead all I did was
add classes to DOM. All the complexities with CSS that's not
fun to deal with is happily abstracted away.
I had to add [markdown](https://github.com/iandinwoodie/github-markdown-tailwindcss) style that was created by someone else,
but that was easy to do.

It is more work to create moving parts yourself, but knowing
how different pieces fit together gives you abiliity to
modify as you would like to. I would like to add sitemap
and RSS feed to my blog.

I have the code for the blog [here](https://github.com/serv/serv.github.io/tree/develop).

Here are some additional resources that I used to create the blog.

- [Configuring github pages](https://docs.github.com/en/github/working-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site#choosing-a-publishing-source)
- [Next.js blog example](https://github.com/vercel/next.js/tree/canary/examples/blog-starter)


Using Next.js To Create Statically Generated Blog


I was struck with a memory leak problem in Node.js application recently.
It is not fun dealing with memory leak problems. Unlike other typical bugs you
face caused by errors in syntax of your code or failures in upstream services,
a memory leak problem defies conventional approaches to squashing the bug.
You need to use unusual tools you don't normally use and you typically will
need more time to solve the problem.

![memory leak](https://i.imgur.com/Skr597z.png)

Let's evaluate some some approaches and tools that you can use to resolve a
memory leak issue in Node.js Applications.

## Heapdump NPM package

[node-heapdump](https://github.com/bnoordhuis/node-heapdump) is an easy to use
NPM package that generates a V8 heap dump of your Node.js application.
You can examine the heap dump with the Developer Tool in the Chrome browser.

- Pros
  - Simple setup
  - Can integrate with your Node.js application and create [a maintenance endpoint](https://github.com/goldbergyoni/nodebestpractices/blob/master/sections/production/createmaintenanceendpoint.md)
  - Easily inspect the V8 heap dump using Chrome Developer Tool
- Cons
  - Only useable if the app is still responsive

My Node.js app was unresponsive because it was executing a blocking function.
The blocking function was preventing the Node.js app from accepting any requests to create a heapdump.

As you can see, there's no silver bullet to investigating and resolving a memory
leak problem. The essence of solving memory leak problem can be described as
simply as 1) get a heap dump 2) inspect the heap dump 3) identify the cause of the
most memory exhausting object creation.
However, there are many different tools and techniques you to solve the
problem and no single way will present a simple solution. Let's continue evaluating
other tools.

## gcore + MDB on Solaris

4 years ago, I solved [another memory leak problem](http://blog.jasonkim.ca/blog/2016/06/03/fixing-memory-leak-on-production-node-application/)
in Node.js application. At that
time, the tool that allows you to inspect memory heap dump of a process, llnode
(I'll present this tool last), wasn't as mature as it is today. And at that time,
I could not use it with the heap dump. I had to use a tool called [MDB](https://docs.oracle.com/cd/E18752_01/html/816-5041/intro-27.html) with [gcore](https://man7.org/linux/man-pages/man1/gcore.1.html).

- Pros
  - You can run gcore even when the Node.js app is unresponsive
- Cons
  - Difficult setup. You need Solaris to run MDB. To learn more, you can read
    this [previous blog post](http://blog.jasonkim.ca/blog/2016/06/03/fixing-memory-leak-on-production-node-application/) I wrote on this topic.

Because setting up Solaris is too cumbersome, I decided to explore using llnode
to inspect the gcore heap dump.

## gcore + llnode

[llnode](https://github.com/nodejs/llnode) is another tool for inspecting gcore heap dump.
I decided to use llnode because I was able to install the tool inside a docker container,
which hosts my Node.js app.

- Pros
  - Moderately easy to setup.
  - llnode is easy to use.
  - You can run gcore even when the Node.js app is unresponsive.

I could not find notable reasons why I shouldn't use gcore + llnode to investigate
the memory issue.

Here are the steps needed to prepare the tools needed to perform your investigation.
I am running Node.js application inside Ubuntu 18 Docker container.

- Go inside the Docker container running the Node.js app.
  `docker exec -it app_name bash`
- Update Ubuntu
  `apt-get update`
- Install lldb
  `apt install lldb-4.0 liblldb-4.0-dev`

  You might see an warning message that reads

  ```
  mount: permission denied
  update-binfmts: warning: Couldn't mount the binfmt_misc filesystem on /proc/sys/fs/binfmt_misc.
  ```

  [You can safely ignore it for our purpose.](https://stackoverflow.com/questions/54951262/binfmt-misc-problems-in-ubuntu18-04-under-docker)

- Install node-gyp
  `npm i node-gyp`
- Install llnode
  `npm install llnode`
- Install gcore
  `apt-get install gdb`
- Run `top` to identify the process number for your Node.js app.
  `36 root 20 0 5700608 4.482g 29724 R 99.7 29.7 100:45.02 node /usr/src/a`
  In this case, the process number is 36.
- Run gcore on the process.
  `gcore 36`
  You might see this error.

  ```
  ptrace: Operation not permitted.
  You can't do that without a process to debug.
  The program is not being run.
  gcore: failed to create core.36
  ```

- To solve the problem in 8, you need to add this to your docker-compose file.

  ```
  cap_add:
  - SYS_PTRACE
  ```

  This is [a good blog post](https://jvns.ca/blog/2020/04/29/why-strace-doesnt-work-in-docker/) on why you need if you are curious.

- Try step 8 again. gcore should work now.
- Inspect the core dump with llnode
  `./node_modules/.bin/llnode node -c core.36`

The process of investigation goes something like this.

- Run `v8 findjsobjects` inside llnode to determine what object is causing the memory leak. You might be wondering how does one tell which object is causing the memory leak.
  There are mainly two ways to nail down the object causing the memory leak.

  - When you have a rapidly growing memory leak, your heap dump presents an extreme version of [Pareto principle](https://en.wikipedia.org/wiki/Pareto_principle). The object will present itself to be occupying a vast majority of memory will be where you will want to investigate. Here's my result of `v8 findjsobjects` demonstrating this effect.

  ```
  ...
  Instances      Size    Name
          109       3488 ContextifyScript
          138       9936 I
          187      13464 (ArrayBufferView)
          213      11928 NodeError
          220      17600 Layer
          226      12656 Node
          226      14456 Entry
          231      11096 Source
          273       6552 CallSite
         3129     250240 Module
        10930     961840 Tok
        16150    1033480 Loc
        30496     975872 (Array)
       336951    8086824 RowDataPacket
      8901702  286210248 Object
     11881688    3728960 (String)
  ```

- When you have a much more slowly growing memory leak, you can't easily tell what
  JS object is responsible for the memory leak. In this case, you have to take 2 heap
  dumps over a period time and see if you see any growth in some JS objects. The
  object growing in the number of instances will be the cause of your memory leak.

- Generic JS primitives (String, Number, Array etc) and Objects are unactionable. Determine what is the JS
  object that is not JS primitives and Objects which appears to be cause
  of the memory leak? In my case, it is `RowDataPacket`.

- By padding logs and metrics around suspicious code, determine where in your code
  causes the `RowDataPacket` object to be created in such number.

You've now found the cause of the memory leak. Unfortunately, finding the cause of the memory leak is not sufficient to resolving the actual memory leak issue.
You still have to apply a remedy to mitigate the cause of the memory leak.
And for that step, good luck.


Investigating Memory Leak in a Node.js Application


When you think about data engineering, people usually think about 
using Java or Python as the primary languages to work with.
The two languages both have a large community backing and many tools 
for handling big data.
While they are fine languages to work with, I want to throw Node.js
into the mix as a serious contender to get the job done.
Node.js is performant enough to do some heavy lifting and has great NPM packages
that help you develop fast. I wanted to share my experience of building data-intensive systems using Node.js.

I've been working at [Coupang](https://www.coupang.com/) for just over a year now. I had the privilege of building the first 
[ETL](https://en.wikipedia.org/wiki/Extract,_transform,_load) 
system for Coupang's advertising platform.
The ETL system began with a modest goal to clean and ingest advertising 
metadata and time series data on a daily basis.
Since the original ETL jobs began every day in the morning, I named
the ETL system, `Morning Star`, referring to planet Venus that appears in 
the east before sunrise.
This is a bit of a misnomer now, given that the system is now running
ETL jobs on a real-time basis. I still kept the name.
Since its inception, Morning Star evolved into a much larger system
that is handling 500 GB of data, 400 million events per day, and the data volume is still growing.

Morning Star needed to provide data to several kinds of users. Services
needed to retrieve appropriate data reliably in a timely fashion.
Analysts asked for a place where they can gather insights about our business. Also, Morning Star needed to provide reliable, normalized data for creating denormalized tables for specific use cases. 

Let's examine the basic data flow of Morning Star. Morning Star's 
infrastructure lives on AWS.
The data flow starts with several Lambda functions that trigger
different jobs for Morning Star by making an HTTP request to the 
load balancer.
The load balancer balances the load in a round robin fashion to 
EC2 instances. Based on the CPU usages, the EC2 instances dynamically
scale up or down to efficiently handle the load.
The EC2 instance that is given the request to handle a job, downloads
the raw files from S3 Data Lake. The raw files in S3 originate
from several other
services. Morning Star cleans, enriches and anonymizes the raw data depending on which job it has to perform. 
The massaged data is then uploaded back to the data lake, where
it is subsequently copied to Redshift.
Redshift is where various other services and users can consume the
data to their liking.

![](https://i.imgur.com/BsZ1i9t.png)

Node.js has been an excellent tool for building the desired system.
Node.js finally has the `async-await` pattern to take advantage of
parallelism without callbacks.
Having worked with callbacks and chained promises, I think
the async-await pattern solved major problems found in the previous
asynchronous programming patterns.
You can program your code spanning vertically rather than horizontally
like the nested callbacks make you do. Your code is easier to read
when it spans vertically.
You can handle errors more precisely than the chained promises pattern where
a single catch block is typically responsible for errors
thrown anywhere during the execution of multiple chained
promises.
You can have a more nimble control flow of your code with `async-await`
because you can use `if-else` statements easier.
The ergonomics of coding was much improved with using async-await.

Speaking of great ergonomics, I should also point out the `stream`
module in Node.js to be a fantastic tool to process large files.
When you work on web services, you typically read and write
discrete chunks of data transactionally. 
On the other hand, it is very common to use stream to move
data from one place another in data engineering.
You often deal with cases where you no longer can load
the entire data onto the memory.
Or you might deal with a source that is continuously generating data that 
needs to be moved to another place in real time. 
That's when you have to stream the data.
Node's stream library helps you stream data reliably and manipulate
the data with ease.
The functions in the stream module are easy to understand and use.
They are well documented with an ample amount of examples, so you can easily
incorporate them into your code.
The stream library plays nicely with the async-await pattern I use
elsewhere as long as I wrap the stream code with a promise.

There are other standard modules in Node.js that makes it 
suitable to do data engineering.
The `child-process` module is used to execute terminal commands,
and it was useful in controlling 
[jq](https://stedolan.github.io/jq/), 
[parallel](https://www.gnu.org/software/parallel/) and 
[Hive Beeline client](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-Beeline%E2%80%93CommandLineShell).
The `fs` module had all the functions I needed to work with the file system,
and `zlib` handle gzipping and gunzipping with stream perfectly.
And if I didn't find an adequate module in the standard Node.js modules,
I found almost all of them in the NPM repository.
This assurance gave me the confidence to push development in Node.js,
and help me iterate faster.

Node.js has some downsides when you try to use it to process data.
I applauded Node.js' async-await to be an excellent pattern to use.
However, this means that you have to write a promise wrapper for
callbacks and pipe functions in order to incorporate it with the
async-await pattern. It's not difficult, but it is tedious.

I also had trouble finding an up to date Hive client for Node.js.
Hive is a widely used data warehousing tool for working with big data.
I was surprised to learn that there's no well maintained 
Hive client available to use on NPM.
For this reason, I had to use Hive's Beeline command line client and
write Node wrapper around it.
I had to install JDK, Hadoop, and Hive as dependencies for Beeline
to run.
There were a lot of unpleasant programming I had to do to make this 
work.

Despite some of the problems with Node.js, it is still a great tool
for data engineering. It has many standard and third-party modules
to help to finish work. Node.js is also fast enough to process large datasets
in real-time. 
I knew that Node.js is great at creating nice web apps and services.
I am glad to know that it is also a fine tool for data engineering.


"Big Data" Engineering with Node.js


I am using [Bluebird](http://bluebirdjs.com/docs/getting-started.html) as 
the main [promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) library for
my Node projects these days.
I frequently use `Promise.all(array)` or `Promise.props(object)` to
resolve multiple promises in an array or an object at once.
This is fine if there is not limit to the number of promises 
your system can handle. Often, this is not the case.
For example, it's not a good idea to resolve 1K requests at once.
This is when you want to use **concurrency** provided by Bluebird.

## Dealing with an Array of Promises

Let's say we used to resolve an array of promises naively this way.

{% highlight javascript %}
function main() {
    let requestUrls = [ '...' ]; // has 1K urls

    let promises = requestUrls.map(reqUrl => {
        return new Promise((resolve, reject) => {
            // ...
        });
    });

    return Promise.all(promises);
}
{% endhighlight %}

Better way to handle this is with concurrency. In this example,
we will resolve 4 at a time. If you want to resolve the promises one at 
a time, you will want to use the concurrency of 1. 

{% highlight javascript %}
function main() {
    let requestUrls = [ '...' ]; // has 1K urls

    return Promise.map(requestUrls, (reqUrl) => {
        return new Promise((resolve, reject) => {
            // ...
        })
    }, { concurrency: 4 });
}
{% endhighlight %}

## Dealing with an Object of Promises

If the number of promises in the object is small, you can use `props` 
to resolve them.

{% highlight javascript %}
function main() {
    let idRequestUrlPairs = { ... }; // has 1K urls

    let promises = {};
    
    idRequestUrlPairs.forEach((value, key) => {
        promises[key] = new Promise((resolve, reject) => {
            // ...
        });
    });

    return Promise.props(idRequestUrlPairs);
}
{% endhighlight %}

Bluebird doesn't have a direct way to resolve an object of promises 
concurrently, but here is a way to do it with `map`. Resolving an object
of promises using concurrency looks something like this.

{% highlight javascript %}
async function main() {
    let idRequestPromisePairs = { ... }; // has 1K urls
    let toBeResolved = {};
    
    await Promise.map(Object.keys(idRequestPromisePairs), key => {
        return new Promise((resolve, reject) => {
            // ...
            return resolve(idRequestPromisePairs[key]);
        })
            .then(result => {
                toBeResolved[key] = result;
            });
    }, { concurrency: 4 });

    // toBeResolved should have resolved promises;
    return toBeResolved;
}
{% endhighlight %}

Resolving Bluebird Promises With Concurrency


[DHH announced earlier in the year that Rails 5.2 will have a brand new component called ActiveStorage](http://weblog.rubyonrails.org/2017/7/15/this-week-in-rails-active-storage-telling-secrets-and-time-travelling/).
ActiveStorage will manage user uploads such as photos directly by Rails.

Since then, a lot of progress has been made to integrate ActiveStorage to
Rails, and you can actually use ActiveStorage today. This blog will explore
the how you can update your Rails app to use ActiveStorage.

**Warning!** You will be using the bleeding edge version of Rails and
there may be unforseen problems caused by it.

## Setting up ActiveStorage for Rails

#### 1. If you are using Rails version before 5.1.14, update Rails to 5.1.14 by updating your Gemfile
{% highlight ruby %}
gem 'rails', '~> 5.1', '>= 5.1.4'
{% endhighlight %}

#### 2. Run `$ bundle update rails`

#### 3. Run `$ rails app:update` and resolve any code differences.

#### 4. Test that updating to `5.1.14` went ok.

#### 5. Now we want to update rails to the latest bleeding edge version. In the
`Gemfile`, add the following lines.
{% highlight ruby %}
git_source(:github) do |repo_name|
  repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
  "https://github.com/#{repo_name}.git"
end
...
gem 'rails', github: 'rails/rails'
gem 'arel', git: 'https://github.com/rails/arel.git'
gem 'bootsnap', '~> 1.1', '>= 1.1.5', require: false
{% endhighlight %}

#### 6. Run `$ bundle update rails`

#### 7. `$ bundle exec rails -v` should display the bleeding edge version,
`Rails 5.2.0.alpha`.

#### 8. Update the application configs. Run `$ bundle exec rails app:update`.

#### 9. When you run, `$ ./bin/rails --tasks`, you should see,
`rails active_storage:install` as an available task.

#### 10. Run `$ ./bin/rails active_storage:install`. This should generate a migration file.

#### 11. Run `$ ./bin/rails db:migrate`. Now our sqlite db should be ready to support ActiveStorage.

## A simple image uploading example using ActiveStorage

Up until now, we've been preparing Rails application to support ActiveStorage.
Now that the application is equipped with ActiveStorage, let us build a
simple feature that allows us to create a post with images
that uses ActiveStorage.

#### 1. Generate post model. `$ ./bin/rails g model post`.

#### 2. Let's add two columns to post table, `title`, `body` in the migration file.
{% highlight ruby %}
# db/migrate/20171114063756_create_posts.rb
class CreatePosts < ActiveRecord::Migration[5.2]
  def change
    create_table :posts do |t|
      t.string :title
      t.text :body

     t.timestamps
    end
  end
end
{% endhighlight %}

#### 3. Let's create a controller for posts resource.
`$ ./bin/rails g controller posts`

#### 4. And add posts resources routes in the `config/routes.rb`.
{% highlight ruby %}
Rails.application.routes.draw do
  resources :posts
end
{% endhighlight %}

#### 5. We need to associate post with images.
{% highlight ruby %}
class Post < ApplicationRecord
  has_many_attached :images
end
{% endhighlight %}

#### 6. Let's add code for index, show and create actions.
{% highlight ruby %}
# app/controllers/posts_controller.rb
class PostsController < ApplicationController
  # we will display post form here
  def index
    @post = Post.new
  end
  # we will create post here
  def create
    post = Post.create! params.require(:post).permit(:title, :body)
    post.images.attach(params[:post][:images])
    redirect_to post
  end
  # we will display post with photo
  def show
    @post = Post.find(params[:id])
  end
end
{% endhighlight %}

#### 7. Add the index view for the post with the following code. This will display upload form on localhost:3000/posts page.
{% highlight erb %}
# app/views/posts/index.html.erb
<%= form_with model: @post, local: true do |form| %>
  <%= form.text_field :title, placeholder: "Title" %><br>
  <%= form.text_area :body %><br><br>
  <%= form.file_field :images, multiple: true %><br>
  <%= form.submit %>
<% end %>
{% endhighlight %}

#### 8. Now let's add a view for show post.
{% highlight erb %}
# app/views/posts/show.html.erb
<%= image_tag @post.images.first %>
{% endhighlight %}

#### 9. Try submitting a photo and you will see the show view displays the image you just submitted.
The image is currently stored locally in a
directory called `storage` in the app root level. But you can configure
this to push the files to cloud file storage systems like AWS S3, Google Cloud
and Azure.

As you can see, ActiveStorage presents a simplied file management system
that is well integrated with Rail's ActiveRecord component.
As ActiveStorage matures, we should see majority of
file management use cases being covered by it.
While there are other file management systems out there for Rails, I suggest that
you consider ActiveStorage before you explore other options.

Also check out [the documentation on ActiveStorage](https://github.com/rails/rails/blob/master/activestorage/README.md).

You can see the final working code on the [github repo](https://github.com/serv/rails-alpha-activestorage-example).

I wanna thank [@jeffreyguenther](https://twitter.com/jeffreyguenther) who
shared his experience of using ActiveStorage on his projects with me.





Using ActiveStorage Today (prior to Rails 5.2 release)


I have been using [ES2015 actively using Babe.js](https://babeljs.io/learn-es2015/)
and I was already convinced that compiling more dev-friendly language to
javascript is useful. One of the langauges that has been praised a lot in
this domain is [Typescript](https://www.typescriptlang.org/index.html).

### Install Typescript

`$ yarn global add typescript`

If you get an error saying `tsc` command is not found, check out this
[SO answer](https://stackoverflow.com/a/40333409/536890) on this issue.

### Linked list in Typescript

Here is a linked list you can start to play with written in Typescript.

```typescript
// index.ts

import { Node } from './node';

export class List {
  head: Node;
  tail: Node;

  constructor() {
  }

  push(val) {
    let node = new Node(val);
    node.next = this.head;
    this.head = node;
  }

  print() {
    let node = this.head;
    while (node !== undefined) {
      console.log(node.val);
      node = node.next;
    }
  }
}
```

```typescript
// list.ts

import { Node } from './node';

export class List {
  head: Node;
  tail: Node;

  constructor() {
  }

  push(val) {
    let node = new Node(val);
    node.next = this.head;
    this.head = node;
  }

  print() {
    let node = this.head;
    while (node !== undefined) {
      console.log(node.val);
      node = node.next;
    }
  }
}

```

```typescript
// node.ts

export class Node {
  val: number;
  next: Node;

  constructor(val) {
    if (!val) {
      return;
    }

    this.val = val;
  }
}

```

Now compile typescript into javascript.

`$ tsc index.ts`

This should generate `index.js`, `list.js` and `node.js`.

Let's run these javascript files.

`$ node index.js`

This should print

```
3
2
1
```

### What I think about Typescript after using it for an hour

Learning Typescript was intuitive and JS I was writing felt more precise
and controlled. This allowed me write less error prone javascript in
a short period time. I should learn more about tooling around Typescript
so that it feels more integrated into the javascript development
workflow.




Setting up Typescript


I worked on upgrading a ~19 KLOC Ember.js codebase.
The previous Ember.js version was 2.3.0.
Now it is 2.12.1. Here, I am going to discuss why I had to upgrade
Ember.js, general approach to solving problems while upgrading a
framework.

For a long time, Ember.js version 2.3.0 served us just fine. The framework
pretty much provided us with everything we needed out of
the box and small flaws here and there didn't bother us. However, over
time, there started to arise several glaring problems with using an
older Ember.js version.

As our app grew larger with some pages generating a large number of
components in a view, we experienced noticeable performance degredation in
DOM rendering. To mitigate this problem, we used an alternative way to
render components, [ember-cli-raw-handlebars](https://www.npmjs.com/package/ember-cli-raw-handlebars) which stripped away much of Ember related functionalities from the view
in order to increase the DOM rendering speed. The trade off for this was
introducing the "none-Ember.js" way of organizing our codebase, which made
our codebase look rather messy.

Using older Ember.js also made it hard to use newer packages out in the wild.
When we wanted to introduce a new package or upgrade an existing package
to a newer version, the Ember framework started to throw cryptic errors
that were hard to debug and fix. The cost of not upgrading Ember.js became
greater than the convenience of relying on an older Ember.js.

## Procedure to upgrade Ember.js

Ember v2.3.0 came out on Jan 17, 2016. A lot has changed since then. Simply
bumping the Ember version to the latest simply will not work. There will
be a lot of noticeable and unnoticeable bugs from doing that. Fixing those bugs
all at once is very difficult. I built a procedure to safely upgrade Ember.js
without creating tons of regressions.

I needed Ember.js upgrade only branch on beta environment. Upgrading Ember.js
of a large endeavor takes a long time. Most likely, you will be asked to develop
features and fix bugs while you are working on upgrade Ember. For that reason,
preparing an isolated Ember.js upgrade only branch on beta will be
useful to allow QA to test the branch while you are working on other features.

I wrote more automated tests. While I was making changes to the Ember.js
upgrade branch, I needed to make sure that I wasn't breaking things while
I fix an issue. Automated tests helped me make sure that I make
safe changes that are tested.

Since your team most likely will QA and review your changes, they will
also need to know how to prepare the development environment for the
Ember upgrade and how to bring it back to their own normal development
environment. I made sure this is clearly described and communicated to
others.

Having some dedicated QA time is also really important. Framework update
touches all aspects of the app, and every core functionalities need to
be tested thoroughly before the change is released on production.

I used binary search like algorithm to bump the versions to the move towards
the latest version.

{% highlight ruby %}
def next_non_erroring_version(start_version, destination_version)
  return start_version if start_version == destination_version

  half_way = (destination_version - start_version) / 2

  if causes_error?(half_way)
    next_non_erroring_version(start_version, half_way - patch_version('0.0.1'))
  else
    next_non_erroring_version(half_way + patch_version('0.0.1'), destination_version)
  end
end

next_non_erroring_version('2.3.0', '2.12.1') + patch_version('0.0.1')
{% endhighlight %}

The method above tries to find the first version closest to the original
version to causes error.




What I learned upgrading a large Ember.js project from v2.3.0 to v2.12.1


I started using Macs since 2011. My first Mac is 13-inch MacBook Pro from
Late 2011. It was a versatile laptop which I was happy to pay a slight
premium for. It served me well through out my school years and
it was a perfect development box.
It was ideal for light amount of gaming to destress as well. Over the years,
the machine started to wear out, it's become sluggish to the point where
I could no longer be productive anymore. It was time to look for another
laptop.

I considered purchasing a 2016 Macbook pro, but the specs were lagging behind
its contemporaries in the same price range. It used to be that I would be
paying 100 to 200 bucks more for the mac when compared to PCs that have
similar specs. Compare to that, now I was looking at paying from 400 to 700
bucks more for the mac. This was unacceptable. I decided to buy a PC instead.
For daily usage of web browsing and gaming, I planned to use Windows 10 and
for software development, I planned to use Ubuntu installed on the same
machine.

I decided to buy [Acer Aspire E 15](https://smile.amazon.com/gp/product/B01DT4A2R4/ref=oh_aui_detailpage_o07_s00?ie=UTF8&psc=1) for $509.99. This was an amazing bargain for a
laptop that has ok cpu, acceptable ram and usable discrete graphics card.
Also, the main harddrive offered is an SSD. The laptop comes with 4 USB ports
and a SD card reader which is perfect for me who wants to get into
photography.

I decided to upgrade 2 things hardware wise. First I wanted to increase my
storage. The machine comes with 250 GB SSD, but given that I want to play
games and store many photographs, I needed a larger harddrive. Also when
I partition the harddrive for Ubuntu, I wanted to leave myself at least 100 GB
for development. I bought [HGST Travelstar 7K1000 2.5-Inch 1TB](https://smile.amazon.com/gp/product/B00B4QESVQ/ref=oh_aui_detailpage_o02_s01?ie=UTF8&psc=1) for $61.99.
That left me with extra 1 TB of storage to play with.

Also another easy upgrade that I decided to make is doubling my memory.
I bought [Crucial 8 GB DDR 2133 MT/s RAM](https://smile.amazon.com/gp/product/B015HQ9TGW/ref=oh_aui_detailpage_o02_s00?ie=UTF8&psc=1) for $44.99.

Installations for these parts are extremely easy, and tutorials can be found
easily online.

And finally dualbooting Ubuntu on the machine requires some preparation, but
it's definitely doable and just having UNIX compatible terminal beats all
the effort required to set it up in the first place.

Overall, preparing a personal development machine that is able to play some
games at $616.97 was totally worth it.



My Macbook Alternative


I recently had to transition to a web team from an API team at work.
It was a difficult transition, but overall, it's a transition that I
am really happy about. I like the new team and the new learning
opportunities transitioning to a new team allows me.
I bet starting a new job probably results in a similar situation, but
with a bigger scope of new work to learn about. Here are some tips
for someone who's starting a new job or transitioning to a work
that will help you to get use to the new tasks as a software engineer.

## 1. Learn the directory structure of the codebase

Learning 100% of the codebase in a short period of time is difficult.
This can be made easier by having someone explain the directory
structure of the codebase for you. Once you understand the context
of the directory sturcture, smaller parts of the codebase can be
made more sense easier.

## 2. Ask questions to the teammembers

You have the experts of the codebase sitting around you. Ask them
questions and get the answers you need.
Have the questions be about things that you can't figure out on
your own. For example, questions about codebase can be figured
out on your own with some struggles, but questions about how
Akamai cache is set up, how deployment works, how testing works,
how oncall process works, etc can't be figured out on your own.
So for these things, you have ask questions.

## 3. Have a vision

You should have a vision for your own work aside from the day
to day work at the team. This was a better advice I've received
from a coworker, and he is right. Doing your day to day job well
makes you a good engineer. If you have a vision about the project
you work on and attempt to improve the product with a vision,
you can be a great engineer. You have to ensure that your vision
is aligned with the product vision, team vision and the company
vision.


Tips for Transitioning to a New Team or a New Job


I recently transitioned off the API team and moved
work on web client team at work. I wanted to share
some things I wish I had known when I was deepdiving
in developing REST API.

## 1. Stability uber alles

You can drop nearly everything on the ground and
focus on restoring stability of the API. No one
will mind.

## 2. Versioned API from the start

APIs are suppose to return consistent data that's
been agreed by the server and the client. There
are times when the server must change the response
body structure to serve the needs of some clients
better. You should not change the response structure
for a given endpoint in such case. You should
version your API so that you can serve the needs
of the old clients and new clients equally well.

Versioning can easily be a topic of conversation
on its own, but here are several considerations
you need to make when you implement versioning.

- Where is versioning going to occur?
  At load balancer or at app?
- Is it going to be done via header or url?
- How are you going to isolate the code change to
  remain within a specific version? Also while
  ensuring that the versioning doesn't make coding
  any more difficult.
- How will you automatically test the newly
  versioned APIs?

## 3. Use API specs

Make use of spec to come to an agreement between
you and the clients before building the API.
You will be able to serve their needs better and
also possibly explain to the clients your limitations.

## 4. Load testing

I didn't realize how important this is until
our production environment was adversely affected
by some feature changes we made without properly
load testing them first. Develop a load testing
strategy for your codebase and be ready to reject
code changes that could affect production
environment negatively.

## 5. Leverage your teammates and clients for advice

Compared to other kind of products I used to work
on developers usually have more correct opinions
about building API.


Easily Overlook Parts on Developing REST API


I had some time over the weekend, and I decided that I would benchmark some
web frameworks for possible backend API to work with in the future. I copied
[how Koa.js does its benchmark](https://github.com/koajs/koa/blob/master/benchmarks/run).
I prepared [Ruby on Rails](http://rubyonrails.org/),
[Restify](http://restify.com/), and [Spark](http://sparkjava.com/) to be
benchmarked. You can see [the code hosted on GitHub here](https://github.com/serv/framework-benchmarks).

Here are the results of the benchmarks. The number represents requests / second.
Higher is better.

| frameworks        | req/s    |
|-------------------|----------|
| ruby on rails     | 48.73    |
| restify           | 8117.63  |
| spark             | 32634.10 |

![Web framework benchmarked](http://i.imgur.com/gOoneii.png)

I knew Ruby on Rails is slow, but I had no idea that it's this slow. And on
the other hand, I knew Java web frameworks are generally fast, but I had no
idea that Spark is that fast! And Restify occupies a sweet spot between
Ruby on Rails and Spark.

We can draw some conclusions from the findings.

- Use Spark for very highly trafficked microservices that requires requests
  to be served at low latency
- Use Restify for all kinds of backend API when you want to leverage a vast
  catalog of Node.js packages and great development tools
- Reserve using Ruby on Rails for product oriented application only.
  I would not recommend Ruby on Rails for backend API that needs to
  serve a large traffic.


Benchmarking Web Frameworks


The last few days at work were rough.
My team was intensely focused on
preparing the production environment to be stable.
We've been having some
serious issues on the production environment.
One of the most serious production issues was a
nasty memory leak.

![](http://i.imgur.com/lEnDrWZ.png)

The graph above displays the memory usage of
12 node applications on our production
environment. The y-axis shows the memory usage
in percentage and the x-axis shows the time
span of 7 days. This narrow sawtooth
pattern on memory usage is extremely bad.
As you can see, all 12 servers
are accumulating usage in memory rapidly.
As a remedy for the memory leak, we had to
periodically restart our servers. This was less
than ideal, but because we had so many other
really high priority items last few weeks, we
just had to bite our tongue and suck it up.
However, the frequency at which we had to
restart our server started to increase.
We had to restart our servers every few days,
then every day, then it came to a point where we
were restarting our servers every 4 to 5 hours.
Thank god we have a globally distributed team
(Croatia, and Argentina), it could've been a
lot tougher without having team members in other
timezones. I can't
emphasize this point enough, and I will take
another opportunity to praise having globally
distributed software development team in another
post in the future.

After some [pitfalls](http://blog.jasonkim.ca/blog/2016/06/02/battle-technical-assumptions/), we fixed the
memory leak. As you can see below, the overall
memory usage stays flat after 13:30 PM after our
memory leak fix was applied.

![](http://i.imgur.com/JnCgcoi.png)

We came up with several different strategies to
fix the memory issue, but the working solution
came down to comparing two memory dumps at
different times and comparing their content.
I used [gcore](http://man7.org/linux/man-pages/man1/gcore.1.html) on a production server to gather a
memory dump soon after the server restart when
the memory usage is around 30%. After around
3 hours when the memory usage for the server
starts to hover around 60%, I took another memory
dump. I used [mdb](https://docs.oracle.com/cd/E18752_01/html/816-5041/chapter-8.html) on a local VM running
Solaris 11 to analyze the two memory dumps.

Here is the memory dump of the production server
at around 30% memory usage.

{% highlight text %}
73ce2207ea9 4643 2 Arguments: 0, length
101b5ab4f71 5519 4 Object: albumId, albumNames, ...
9719767799 6070 2 Object: localeCode, value
971977c371 6966 1 Object: entry
1102bfc1dc31 8775 2 Object: id, genreIds
971977b021 9086 2 Object: ids, href
971973e051 9430 3 Array
3015e01 16203 0 Object
971974c9c9 19854 1 Object: nr_exclusive_duration_millis
9719710199 19917 13 TraceSegment: name, transaction, ...
971971cb11 20186 7 Timer: state, touched, duration, ...
342d8d9 49769 0 Array
971971cbb9 59489 1 Array
971970f1c9 71743 2 Array
OBJECT #OBJECTS #PROPS CONSTRUCTOR: PROPS
{% endhighlight %}

And here is the memory dump of the production
server at around 80% around 4 hours after
the server restart.

{% highlight text %}
1102bfc1dc31 8775 2 Object: id, genreIds

...

110879a2c9 14099 6 ExponentialBackoffStrategy: ...
1108779561 15593 3 Array
45357fdf31 17059 2 Object: ids, href
3015db1 34680 0 Object
110875b221 35287 4 Object: albumId, albumNames, ...
110875b2c9 38401 1 Object: entry
110875aff1 38481 2 Object: localeCode, value
1108708d51 198161 0 Array
1108706ae1 202984 1 Array
1108706a81 206039 1 Object: nr_exclusive_duration_millis
11087046d9 209896 13 TraceSegment: name, transaction, ...
1108704801 223481 7 Timer: state, touched, duration, ...
1108706aa1 507224 2 Array
OBJECT #OBJECTS #PROPS CONSTRUCTOR: PROPS
{% endhighlight %}

The most notable cause of the memory increase is
`TraceSegment: name, transaction, ...`.
You can see that the number of `TraceSegment`
object count increased by 10 folds going
from 19917 to 209896. You can also see some
other object such as `Timer` and `Array`
increasing. This is because those are properties
of the `TraceSegment` object.
Contrast this number with the genre-tags object
that stayed exactly the same
`2 Object: id, genreIds` at 8775. That object
is a json file parsed into memory at startup
of the app and it does not increase. This
suggests to me that the memory dump is indeed
accurate, and Node retains memory steady on
correctly coded objects.

It turns out `TraceSegment` objects are created
by [NewRelic](https://github.com/newrelic/node-newrelic/blob/1e7bbbaf34a15f0bb35aca63b0d8a3cfa2669d27/lib/transaction/trace/segment.js). The fact that
I wasn't able to recreate the severity of
memory leak convincingly on the development
environment from previous attempts also tells
me that the memory leak was caused by something
happening on the production environment.

We get some benefits of having NewRelic monitoring, but disabling NewRelic on
production was no-brainer at this point.
We had to do it. We all acknowledged that
NewRelic is not the initial cause of the memory leak.
There is still something in our code that is
behaving naughtily with NewRelic, but we
got to put our the damn fire on production
first. So we did it. We disabled NewRelic
on production and the memory usage stopped
climbing and it remains very steady for now.

I want to thank [Yunong Xiao for sharing
Netflix's experience on debugging Node.js in
production](https://yunong.io/2015/11/13/debugging-node-js-in-production/). It informed me about
useful tools in the wild and taught me how
I can use those tools to find the cause
of the memory leak. I also want to thank
[Bryce Neal for writing a detailed tutorial on setting up Solaris to use MDB](http://bryce.is/writing/code/mdb_v8/vm/virtualbox/omnios/2016/02/06/mdbv8-getting-started.html).


Fixing Memory Leak on Production Node.js Application


I was burned pretty bad yesterday. I’ve told myself many times to battle assumptions about feature and product requirements, and that product managers cannot give 100% oversight on what I am working on. Now I am going to add another kind of assumption I will question every time. The kind I am talking about is a technical assumption.

Just like assumptions you make about a feature or a product, as a developer, making wrong technical assumptions can cost you dearly. Making technical assumptions get you started with the development faster, but you are playing a Russian roulette with yourself if you don’t have adequate technical planning to accompany your development process.

Wrong technical assumptions can lead you to think that you are able to accomplish something when it is not technically feasible. I learned this the hard way yesterday at work.

![](http://i.imgur.com/UtV84yu.png)

Currently we are having some nasty memory leak problem on the production environment coupled with unstable Redis storage and periodic slow down of upstream services.
My team is working really hard to resolve the issue.
After researching how to debug production Node systems, I decided that I would try getting some memory snapshots on a production server, and examine what is causing the memory increase.
I chose to use [heapdump](https://github.com/bnoordhuis/node-heapdump) to get V8 heap snapshots, and use Chrome’s developer tools’ profiler to examine the data.

![](http://i.imgur.com/tOL9Xir.png)

In theory this was just perfect.
In practice, I made some unsound technical assumptions which resulted in a full day wasted.
Our local development environment are running Node 4.2.4 and the latest NPM 3.x. I was able to install Heapdump package no problem locally. But on the production environment, we are still running NPM 1.x!
I couldn’t find a way to install Heapdump on production. This problem itself can be solved by updating NPM obviously, but we haven’t fully tested how NPM 3 would work with our deployment process to our beta, load, and production environment.
Also any infrastructure change must be orchestrated through our TechOps team, and that typically takes several days to coordinate all efforts.

Wrong technical assumptions are just as deadly as feature/product assumptions.
I will plan out more granular and incremental steps to determine how each development process will be carried out to meet the acceptance criteria of the feature story from now on.


Battle Technical Assumptions


I solved an interesting problem at work
recently, so I wanted to discuss that here.

At Rhapsody, we handle millions of tracks and
tracks are categorized under hundreds of genres.
Genres are related to other genres by being
a parent or a child to another genre. A genre
may have up to 1 parent genre and have as many
children genres. Some top level genres such as
rock, pop, and hip hop do not have any parents.
It looks like a tree. The figure below is an
simplified example of how a genre tree looks
like.

![](http://i.imgur.com/ynbGcss.png)

Some genres have tags associated with them.
Tags are objects that describes what kind of
quality a piece of content has. Drake’s latest
album, Views, feels summery, catchy, soulful
and reflective. Then Views will have tags like
summer, pop, soulful and reflective.

Now there was a problem where we had a way to
retrieve genres from tags, but we could not
retrieve tags from genres. As a temporary solution
to the problem, we decided that we would build and
store the mapping of going from genres to tags as
a static JSON file.

Genre tags are similar to genres themselves, and
genre tags also have a tree data structure. Each
tag has a parent tag and can have many children
tags.

In order to go through the entire genre tags tree,
I decided that I would implement recursive depth
first traversal method in ruby to make API calls to
retrieve data on all genre tags and its children.

There are different parts to the code to accomplish
this, but let’s just look at the meatest part of
the code.

{% highlight ruby %}
def self.get_children(tag, array)
  if tag['childrenTagList'].nil? || tag['childrenTagList'].empty?
    array << { 'id' => tag['id'], 'genreIds' => tag['genreIds'] || [] }
    return array
  end

  array << { 'id' => tag['id'], 'genreIds' => tag['genreIds'] || [] }
  tag['childrenTagList'].each do |t|
    array << get_children(get_tag(CATALOG, t['id']), [])
  end

  array
end
{% endhighlight %}

The code accepts `tag` which is a string and
`array` which contains pairs of tag_id and
genre_id. I better check for the terminating
condition. The terminating condition is to check
to see if the tag has any children tags. If it
doesn’t have any children tags, I proceed to
populate the array with the tag-genre pair and
return the array. Otherwise, I should still
populate the array with the tag-genre pair. But in
addition to that, I have to recursively call
`get_children` for all children tags.
`get_children` is responsible for requesting
metadata of children tags given a tag ID, using
[Faraday](https://github.com/lostisland/faraday).
The response body in JSON format is parsed by
[Oj](https://github.com/ohler55/oj).


Using Recursive Depth-first Traversal to Make API Calls to the Object’s Children


I waste a lot of time not coding. Typical work involves many activities,
that are process related chores, meetings and etc, so only a fraction of time
can be spent on coding and working on a feature. I find it important that
you should try to squeeze most out of this time as possible.

I try to understand the product feature and its use cases before I begin
coding on a feature.
In an ideal world, your product owner will write a perfect story that
doesn't require any edits whatsoever. This never happens. Product owners are
humans and they will usually write an insufficient story
when the stories are sufficiently large and complicated.
Developers must be able to wear the product owner's hat to come up with the
rest of the acceptance requirements for a story.
Try to come up with what makes sense and what doesn't make sense about the
story. While you are trying to come up with what makes sense,
bring it up with your product owner to see if some additional acceptance
criteria needs to be added. When things don't make sense in the story, also
bring it up with product owner sooner rather than later.

Here's another thing I do before I begin coding.
I spend a good length of time thinking about the software design of the feature.
Explore how each acceptances in the story can be mapped to your software design.
Your code builds on top of other code you wrote.
Code you write has inertia. Early on during the coding phase, you can make
small and large changes without much trouble. You are nimble when your code
for the feature isn't large yet. But when you have to change the software design
later on down the road, changing your existing code that has grown larger
is way more difficult. There are many more moving parts and each changes
you have to make in your code will need to make multiplicative changes across
your code base. Also engage your teammates in brainstorming session and
pick their brains on how they think the code should look.

After you are completed with coding, have a rigorous code review with your
coworker. I was wrong about this before. I have been doing more lenient
code review that left quite a room for disagreement. Be respectful in your
tone and watch out for strong language. Messages do not get understood
best over the web, especially when that message is about criticizing your code.
This code review should be contrasted with nitpicking. It's not about trying
to inject your preference for the code. It should be about writing good code.


Time Saving Tips for Software Development for Developers


I find deployments hard. But why? I conjectured a few reasons why.

1. You don't practice setting up deployments too often.
2. When you do the setup, you usually mindlessly copypasta the command
from internet.
3. And you finally make it work without any deep understanding, well
you never try to relearn what you've done.
4. Deployment is actually a hard subject that touches on many different
complex systems.
5. When something goes wrong, it's hard to fix or undo. And you might
compound the problem by trying to fix it by copypasta-ing even more crap.
6. You might not even be aware of the problem at all. You could've done
something an hour that fucked up the deployment process, but you might
be trying to undo something you did 10 minutes ago. Diagnosis of the
problem is really hard for deployment.

What is the remedy for these problem?

1. You have to practice deployment like everything else. In order to
practice something you need a meticulous guideline on how to practice.
There for document everything!
2. Understand each steps you are performing. Never just copypasta
something. Even better, understand the system you are using.
3. Even after you make something work, go back and review what you've
just done.
4. Learn the system you are using. Again, don't just copypasta.

I find that 5 and 6 are rather hard problems. You need really good
understanding of the system in order to debug some problem.
Otherwise, you just end up googling same shit over and over again.
Maybe I'll come up with some solution for them later down the road.


The Deployment Minefield


[Phoenix](http://www.phoenixframework.org/) is a promising
[Elixir](http://elixir-lang.org/) web framework that is fast and productive.
I heard about Phoenix through a coworker a couple of weeks
ago and started playing with it.
Then stopped for a while, until I heard that the framework finally
hit [1.0 on Hacker News](https://news.ycombinator.com/item?id=10135825).
There were many praises for the framework and success stories
that I thought I had to try it again.

This is a summary of what I did to set up Phoenix 1.0 on
Mac OS X Yosemite (Version 10.10.5). I also included how
I solved some odd errors that appeared during installation process.

1. `$ brew install elixir`
2. `$ mix local.hex`
3. `$ mix archive.install https://github.com/phoenixframework/phoenix/releases/download/v1.0.0/phoenix_new-1.0.0.ez`

    In this step, you might see this error.

    ```
    ** (Mix) Could not access url https://github.com/phoenixframework/phoenix/releases/download/v1.0.0/phoenix_new-1.0.0.ez, error: {:failed_connect, [{:to_address, {'github.com', 443}}, {:inet, [:inet], :nxdomain}]}
    ```

    To solve this issue, you need to ensure that `~/.mix/archives directory` has write permission.

    Run `$ sudo chmod a+rw ~/.mix/archives/` then try running #3 again.

4. Go into the directory where you want your Phoenix app to live.
5. `$ mix phoenix.new hello_phoenix`
6. Say yes to `Fetch and install dependencies? [Yn] y`

    You might see an error during this process. And most likely, it's happening because your Phoenix project directory doesn't have write permission.

    Run `$ sudo chmod -R a+rw hello_phoenix`.
    Then try running #5 again.
7. Run `$ npm install` with node > 0.12.0
8. Ensure that you have postgresql running.
9. Run `$ mix ecto.create`

    You might get this error.
    ```
    ** (Mix) The database for HelloPhoenix.Repo couldn't be created, reason given: "psql: FATAL:  role \"postgres\" is not permitted to log in\n".
    ```

    Run `$ psql postgres`

    In the psql, run `CREATE ROLE postgres;`

    Try running `$ mix ecto.create` again.

    You might get this error.
    ```
    ** (Mix) The database for HelloPhoenix.Repo couldn't be created, reason given: "psql: FATAL:  role \"postgres\" is not permitted to log in\n".
    ```

    In the psql, run `ALTER ROLE postgres LOGIN;`

    Try running `$ mix ecto.create` again.

    You might get this error.
    ```
    ** (Mix) The database for HelloPhoenix.Repo couldn't be created, reason given: "ERROR:  permission denied to create database\n".
    ```

    In the psql, run `ALTER ROLE postgres CREATEDB;`

    Now try `$ mix ecto.create` again

    You should see `The database for HelloPhoenix.Repo has been created.`

10. Run `$ mix phoenix.server`
11. Go to http://localhost:4000/

![](http://i.imgur.com/o7mD8Fv.png)


Setting up Phoenix 1.0 on Mac OS X Yosemite (With Troubleshooting Steps)


"Oh, that's easy."

We say this often when we help beginners with coding. I think we say this
for several reasons, but one big reason is to encourage the beginner that
he or she can solve the problem by reducing the perceived difficulty of the
problem. However, in reality, this probably has an adverse effect.
First of all, it's condescending. It puts the beginner's confidence down.
He or she must be thinking, "If this is so easy, why can't I do it?"
You might even be wrong that it's an easy problem. There may be details of
the problem you haven't noticed yet that genuinely difficult problem
look very easy.

I believe that you can omit saying "That's easy," completely and help
out someone effectively. Just treat problems beginners have as a problem
you and your colleagues have. Don't belittle the beginner with phrases like
"That's easy" and just focus on what the problem is and helping them solve
the problem.


Don't say something is easy


I found out that someone who is a friend and a coworker passed away years ago,
and I didn't know about it until today. I felt bad that I hadn't tried to
reach out to him more when I had the chance. He was a sincere guy who was
always positive. It makes me angry that someone so good passed away when he
was just 23. All the possibilities and dreams he might have had, they are all
gone. He didn't even get a shot at life.

People talk about unfairness a lot. They complain about how things are so
unfair in life. I do too. In a way, this game called life is rigged. Some people
really have many advantages in life while other don't. You can rightly
complain about how life is unfair because it really is so. But what does that
get you? Nothing. It'll in fact mess you up even more. It'll weaken you and
keep you stagnant. If you notice you are behind, think about how you are
going to catch up.

In fact, a vast majority of people even if they are way way behind others,
if they are living a life, working hard, studying, striving to better
himself and others, they are trying. Everyday, you get another chance at life.
A chance some others don't have.


Unfair


We know Bower is great for managing frontend packages such as jQuery and Bootstrap.
I noticed that it's not obvious how I can use Bower with Rails and Heroku.
Here is how I set up Bower to work with Rails 4 app running on Heroku.

My local system setup looks like this.

- OS X Yosemite V 10.10.3
- ruby 2.2.1p85 (2015-02-26 revision 49769) [x86_64-darwin14]
- Rails 4.2.3
- Bower 1.4.1
- Node v0.10.38

## 1. Prerequisites

I'm gonna assume that you have some basics down already.

- Install Ruby and Ruby on Rails.
- Install Node and have npm working.
- Install Bower

## 2. Configure npm package and bower package

Run `npm init` to create `package.json` file with correct npm
configuration for your app.

Now run `npm install bower --save`. This will include bower as a
dependency for your npm package.json.

Whenever you install a npm package, add `--save` to automatically
add the package as the dependency.

Run `bower init` to create `bower.json` file with correct bower
configuration for your app.

Now install your packages running `npm install [pakcage-name] --save`.

Again, whenever you install a bower package, add `--save` to automatically
add the bower package as the dependency.

## 3. Make Bower work with Rails

Let's specify where bower packages are installed on the Rails app.

Create a file called `.bowerrc`, and add

{% highlight json %}
{
"directory": "vendor/assets/bower_components"
}
{% endhighlight %}

`vendor/assets/bower_components` is where bower packages will be installed.

At this point, let's edit `.gitignore`, so we don't commit npm packages or bower packages.
At the end of `.gitignore` file, add

{% highlight text %}
node_modules
vendor/assets/bower_components
{% endhighlight %}

In `config/application.rb`, add:

{% highlight ruby %}
config.assets.paths << Rails.root.join('vendor', 'assets', 'bower_components')
{% endhighlight %}

This will ensure that Rails picks up asset files from where bower packages are
installed.

If you wanna start using Bootstrap for example, in the `app/assets/stylesheets/application.css`,
you can add:

{% highlight js %}
\*= require bootstrap/dist/css/bootstrap
{% endhighlight %}

And in `app/assets/javascript/application.js`,

{% highlight js %}
//= require bootstrap/dist/js/bootstrap
{% endhighlight %}

Make sure restart the Rails app to see the bower packages working with Rails.

## 4. Use Bower on Heroku

At this point, Bower should be working well with Rails on your local environment.
However, when you try to use it on Heroku, it'll error out while it
is trying to precompile assets.

This part, I am following some steps taken in [Anthony Smith's post](https://coderwall.com/p/6bmygq/heroku-rails-bower).

Configure Heroku to use ddollar's multi-buildpack:

`heroku config:add BUILDPACK_URL=https://github.com/ddollar/heroku-buildpack-multi.git`

Create a file called: `.buildpacks` and add:

{% highlight text %}
https://github.com/heroku/heroku-buildpack-nodejs
https://github.com/heroku/heroku-buildpack-ruby
{% endhighlight %}

Make sure that the ruby buildpack is the last one in the list.
This will allow you to access the Rails console when running heroku run console.

In the `package.json`, add a postinstall script. It should look like this.

{% highlight json %}
{
...

"scripts": {
"postinstall": "./node_modules/bower/bin/bower install"
}

...
}
{% endhighlight %}

Now push to Heroku!

`git push heroku master`

`npm install` will take a while to finish.


Setting up Bower on Rails 4 for Heroku


This isn't the first time [someone has considered metaprogramming in ruby to be harmful](https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=metaprogramming%20harmful),
and it won't be the last. Metaprogramming is useful when you are programming
reactively against external changes that you have to account for with your
code. An example I like to use is the time when I had to create attributes
for an object based on external json object. So even if the json object
changes its attributes, the ruby program is able to account for the changed
attributes.

However, in the real world, proper use of metaprogramming is infrequent.
I would even venture to say that, if you are considering doing something
with metaprogramming in Ruby, stop and try to solve it without metaprogramming.

Metaprogramming is not obvious and even to the author himself, it'll become
less obvious over time. Metaprogramming often ends up ugly. This is
certainly true for someone else reading the code. It is harder to debug.
You end up having a hard time finding the code using `grep` or the text editor because
the code is obscured via metaprogramming.

If you must metaprogram, document it well. Let your colleagues know what you
are doing. Be very open about how your metaprogramming works. And it never
hurts to get a second opinion on how you can avoid doing this altogether.


Metaprogramming in Ruby should be avoided if possible


[Attributa](https://github.com/serv/attributa) is a node.js package
that helps you to assign multiple attributes
in just one line.

## Problem

Did you ever have to do this?

{% highlight js %}
function User(data) {
  this.username = data.username;
  this.password = data.password;
  this.address = data.address;
  this.email = data.email;
  // ...
  // After a dozen more attributes...
  this.privacySetting = data.privacySetting;
}
{% endhighlight %}

## Solution

You can now do this instead!

{% highlight js %}
function User(data) {
  attributa(this, data);
}
{% endhighlight %}

The installation instructions and docs can be found in
[the repository](https://github.com/serv/attributa).


attributa: a simpler way to assign values to attributes in node.js


1. [Code: The Hidden Language of Computer Hardware and Software](http://www.amazon.com/Code-Language-Computer-Hardware-Software/dp/0735611319/ref=sr_1_1?ie=UTF8&qid=1420415680&sr=8-1&keywords=Code%3A+The+Hidden+Language+of+Computer+Hardware+and+Software)
by Charles Petzold

2. [The C Programming Language, 2nd Edition](http://www.amazon.com/C-Programming-Language-2nd/dp/0131103628/ref=sr_1_1?ie=UTF8&qid=1420415769&sr=8-1&keywords=C+Programming+Language+%282nd+Edition%29)
by Brian W. Kernighan and Dennis M. Ritchie

3. [Algorithms](http://www.amazon.com/Algorithms-4th-Robert-Sedgewick/dp/032157351X/ref=sr_1_1?ie=UTF8&qid=1420419952&sr=8-1&keywords=algorithms) by Robert Sedgewick and Kevin Wayne


3 technical books I want to read for 2015


Since last July 2014, I started learning about
[Backbone.js](http://backbonejs.org/) for
[Soundtracking.com](https://soundtracking.com/)'s web client.
To learn more about Backbone.js and REST API/Client architecture,
I decided that I would spend the rest of 2014 working on creating a
[Delicious](https://delicious.com) clone.

Unfortunately, it is only about 50% finished, and I don't intend to complete it.
I achieved the initial goal of understanding these things and I feel that
I learned what I needed from this project. Big things I learned from this
project were:

- Creating REST API
- Creating single page app using Backbone.js
- Rolling out my own OAUTH2

You can check the project repos here.

- [Backbone.js client](https://github.com/serv/supurl)
- [Ruby on Rails API / Backend](https://github.com/serv/supurl-api)


Learning from Suprul


Note: this is still incomplete.

## Prerequisites

- Minimal understanding of what Dokku, DigitalOcean and Ubuntu ara
- Already has DigitalOcean account
- Knows basics of SSH

## Creating DigitalOcean Droplet

1. Sign into DigitalOcean and click on the button, "CREATE" on the left sidebar.
2. Insert a good Droplet Hostname and select the size of the droplet that suits
your application.
3. Select the region. I'm choosing the default one, which is NY3.
4. In "Select Image" section, click on "Linux Distributions" tab.
5. Click on Ubuntu and select **14.10 x64**.
6. We will create and add a new SSH key manually. DO NOT select any SSH key
displayed under "Add SSH Keys" section.
7. Click on "Create Droplet"

## Logging into the new server for the first time

1. You should have received an email that contains your host IP address,
username and the password. Keep the password as secret! Username is `root` by
default.
2. Log into the host server using the username. `$ ssh root@123.123.123.123`
Replace `123.123.123.123` with your host ip address.
3. Type `yes` to the question `Are you sure you want to continue connecting (yes/no)?` and press Enter.
4. Copy and paste the root password that was emailed to you.
5. The server will prompt you to update your password. Follow the instructions
given to you and update the password.

## Creating public key on your local machine

1. On your local computer's terminal, run `$ cd ~/.ssh`
2. Run `$ ssh-keygen -t rsa -C "your_email@example.com"`
3. To the question, `Enter file in which to save the key (/Users/you/.ssh/id_rsa): `, type in the host name of the DigitalOcean droplet you just create. Just
good practise to have the same name.
4. Enter the password for the pubkey you just created. This doesn't have to
be the same as the root password you changed earlier.
5. Run `$ cat key-you-created.pub` to check that pubkey was created successfully.
6. Copy the result of `$ cat key-you-created.pub` in the local machine.

## Adding the new SSH Key in DigitalOcean

1. Log into DigitalOcean
2. Click on "SSH Keys" on the left hand side.
3. Click on "Add SSH Keys" on the top right hand side.
4. Paste the pubkey content in the form and submit it.

## Configuring your local machine to use this specific new pubkey
when connecting to the server

1. In the local machine, run `$ vi ~/.ssh/config`
2. Add the following lines.

{% highlight text %}
Host [some-host-name-nickname]
  Hostname     [123.123.123.123].
  User         root
  IdentityFile ~/.ssh/[key-you-created]
{% endhighlight %}

3. Replace all the data in `[]`.

## Authenticating yourself against the server

1. In the server console run `$ cd ~/.ssh`
2. Again, in the server, run `$ echo "ssh-rsa [paste the content of key-you-created.pub]" >> authorized_keys`.
3. Run `$ cat authorized_keys` and check that your local machine's pubkey is
in the list of authorized_keys in the server.

## Phew, take a breather

If everything went right, you should be able to do, `$ ssh root@some-host-name-nickname`.

## Installing Dokku

1. I am largely following [the official installation doc for Dokku](http://progrium.viewdocs.io/dokku/installation).
2. On the server, run `$ wget -qO- https://raw.github.com/progrium/dokku/v0.3.7/bootstrap.sh | sudo DOKKU_TAG=v0.3.7 bash`
3. `$ cat ~/.ssh/supurl-production.pub | ssh root@104.131.160.133 "sudo sshcommand acl-add dokku $USER"`
4. `$ git remote add dokku dokku@supurl-production:supurl-production`

## Troubleshooting various errors

{% highlight text %}
git push dokku master
Counting objects: 1012, done.
Delta compression using up to 8 threads.
Compressing objects: 100% (772/772), done.
Writing objects: 100% (1012/1012), 109.41 KiB | 0 bytes/s, done.
Total 1012 (delta 526), reused 298 (delta 137)
-----> Cleaning up ...
-----> Building supurl-production ...
Ruby app detected
-----> Compiling Ruby/Rails
-----> Using Ruby version: ruby-2.0.0
-----> Installing dependencies using 1.5.2
New app detected loading default bundler cache
!
!     Command: 'set -o pipefail; curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://s3-external-1.amazonaws.com/heroku-buildpack-ruby/ruby-2.0.0-p481-default-cache.tgz -s -o - | tar zxf -' failed unexpectedly:
!
!     gzip: stdin: unexpected end of file
!     tar: Child returned status 1
!     tar: Error is not recoverable: exiting now
!
To dokku@supurl-production:supurl-production
! [remote rejected] master -> master (pre-receive hook declined)
error: failed to push some refs to 'dokku@supurl-production:supurl-production'
{% endhighlight %}

Build pack error.
https://github.com/heroku/heroku-buildpack-ruby/issues/304


{% highlight text %}
I, [2014-12-09T01:09:50.397656 #12]  INFO -- : listening on addr=0.0.0.0:5000 fd=10
E, [2014-12-09T01:09:50.403558 #12] ERROR -- : could not connect to server: No such file or directory
Is the server running locally and accepting
connections on Unix domain socket "/var/run/postgresql/.s.PGSQL.5432"?
(PG::ConnectionBad)
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:888:in `initialize'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:888:in `new'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:888:in `connect'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:568:in `initialize'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:41:in `new'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/postgresql_adapter.rb:41:in `postgresql_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:435:in `new_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:445:in `checkout_new_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:416:in `acquire_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:351:in `block in checkout'
/app/vendor/ruby-2.1.5/lib/ruby/2.1.0/monitor.rb:211:in `mon_synchronize'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:350:in `checkout'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:265:in `block in connection'
/app/vendor/ruby-2.1.5/lib/ruby/2.1.0/monitor.rb:211:in `mon_synchronize'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:264:in `connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_adapters/abstract/connection_pool.rb:541:in `retrieve_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_handling.rb:113:in `retrieve_connection'
/app/vendor/bundle/ruby/2.1.0/gems/activerecord-4.1.6/lib/active_record/connection_handling.rb:87:in `connection'
./config/unicorn.rb:15:in `block in reload'
/app/vendor/bundle/ruby/2.1.0/gems/unicorn-4.8.3/lib/unicorn/http_server.rb:519:in `call'
/app/vendor/bundle/ruby/2.1.0/gems/unicorn-4.8.3/lib/unicorn/http_server.rb:519:in `spawn_missing_workers'
/app/vendor/bundle/ruby/2.1.0/gems/unicorn-4.8.3/lib/unicorn/http_server.rb:140:in `start'
/app/vendor/bundle/ruby/2.1.0/gems/unicorn-4.8.3/bin/unicorn:126:in `<top (required)>'
/app/vendor/bundle/ruby/2.1.0/bin/unicorn:23:in `load'
/app/vendor/bundle/ruby/2.1.0/bin/unicorn:23:in `<main>'
{% endhighlight %}

`$ dokku postgresql:create supurl-production`

http://dev.housetrip.com/2014/07/06/deploy-rails-and-postgresql-app-to-dokku/

`dokku run APP_NAME rake db:seed `


{% highlight text %}
# installing the plugin
cd /var/lib/dokku/plugins
git clone https://github.com/Kloadut/dokku-pg-plugin postgresql
dokku plugins-install

# Create and setup the db
dokku postgresql:create <db-name>

# Once you've deployed your app once, you can link the db to the app like:
dokku postgresql:link <app-name> <db-name>
{% endhighlight %}


{% highlight text %}
remote: runtime: panic before malloc heap initialized
remote: fatal error: runtime: cannot allocate heap metadata
To dokku@supurl-production:supurl-production
! [remote rejected] master -> master (pre-receive hook declined)
error: failed to push some refs to 'dokku@supurl-production:supurl-production'
{% endhighlight %}

https://github.com/docker/docker/issues/1555
http://stackoverflow.com/a/17174672/536890


Setting up Dokku on DigitalOcean using Ubuntu 14.10 x64


## Bitwise operators

- `&`: Bitwise AND
- `|`: Bitwise OR
- `^`: Bitwise XOR
- `\>\>`: Shift right
- `>>>`: Unsigned shift right
- `<<`: Shift left
- `~`: One's complement (unary NOT)

You can use these bitwise operators for the following data types.

- `int`
- `long`
- `short`
- `char`
- `byte`

Bitwise operators cannot work on ...

- `boolean`
- `float`
- `double`

## Bitwise operations

![from Java: A Beginner's Guide](http://i.imgur.com/Pic8E71.png)

From [Java: A Beginner's Guide](http://www.amazon.com/Java-Beginners-Guide-Herbert-Schildt/dp/0071809252/ref=sr_1_1?s=books&ie=UTF8&qid=1417377646&sr=1-1&keywords=Java%2C+A+Beginner%27s+Guide)

## Example 1: Getting upper case for a letter

{% highlight java %}
public class UpCase {
public static void main(String args[]) {
char ch;

    for(int i=0; i < 10; i++) {
      ch = (char) ('a' + i);
      System.out.print(ch);

      ch = (char) ((int) ch & 65503);

      System.out.print(ch + " ");
    }

}
}
{% endhighlight %}

The value 65,503 used in the AND statement is the decimal representation of 1111 1111 1101 1111. Thus, the AND operation leaves all bits in ch unchanged except for the 6th one, which is set to 0.

## Iterating through bits

{% highlight java %}
for(int t = 128; t > 0; t = t/2) {
// ...
}
{% endhighlight %}

## Example 2: Getting lower case for a letter

{% highlight java %}
public class LowCase {
public static void main(String args[]) {
char ch;

    for(int i = 0; i < 10; i++) {
      ch = (char) ('A' + i);
      System.out.print(ch);

      ch = (char) ((int) ch | 32);

      System.out.print(ch + " ");
    }

}
}
{% endhighlight %}

## Shift operators

- `<<`: left shift
- `>>`: right shift
- `>>>`: unsigned right shift


Java Bitwise Operations


## tl;dr

- Installing Java 8 on Mac OS X Yosemite
- Updating Eclipse Kepler to work with Java 8
- Fixing `Error: Could not find or load main class packageName.ClassName`
- Fixing `No JREs in workspace compatible with specified execution environment: JavaSE-1.8`

## Dirty Java 8 and Eclipse

Making Eclipse Kepler to work with Java 8 on Mac OS X Yosemite wasn't all that obvious. I guess this
is [why people hate Eclipse so much][1]. Here are the steps I followed to make
Eclipse Kepler to work with Java 8.

1. Visit this [page][2].
2. Follow the section, `Installing Java™ 8 support for Eclipse Kepler SR2`.
3. Install [Java SE Development Kit 8][3].
4. Follow the subsection, `Use the Eclipse Marketplace to add Java™ 8 support to your Eclipse Kepler SR-2 package`.
5. If you attempt to create a new project, a new package then a new class with
a `main` function and run it, you might get an error that goes: `Error: Could not find or load main class packageName.ClassName`
6. One of the top hits on Google for this error was this [Stackoverflow page][4].
7. When I right-clicked on the project and tried to edit the build path, I kept on
seeing `No JREs in workspace compatible with specified execution environment: JavaSE-1.8`.
8. To fix this, I found another [Stackoverflow page][5] that suggested a solution.
9. The problem is how the fuck do you know where Java 8 is? Thankfully, there a
command in terminal you can run `/usr/libexec/java_home -V`.
10. Look for `"Java SE 8"	/Library/Java/JavaVirtualMachines/jdk1.8.0_25.jdk/Contents/Home`
11. Go through the [second Stackoverflow page][5] with the newly found location
of Java SE 8.
12. You should be able to run the example program now.

[1]: http://www.ihateeclipse.com/
[2]: http://www.eclipse.org/downloads/java8/
[3]: http://www.oracle.com/technetwork/java/javase/downloads/index.html
[4]: http://stackoverflow.com/questions/22755551/eclipse-error-could-not-find-or-load-main-class-hello
[5]: http://stackoverflow.com/questions/22619262/upgrade-eclipse-to-java-8


Updating Eclipse Kepler and Fixing its Woes on Mac OS X Yosemite


## tl;dr

I bought 2009 Pontiac Vibe GT after researching new and used cars for close to a year.
Buying a used car from a private seller is more complicated than buying a new car or
a used car from a dealer, but you will probably get a better deal.

![](http://i.imgur.com/D3CfyMT.jpg)

I finally got a car! I wanted one for a long time, but it never really made sense financially
until now. The car I got is 2009 Pontiac Vibe GT with about 86,000 miles.
I got a pretty good bargain on the car and I am really pleased with the purchase.
As a celebratory act, I went to get a large load of grocery. Thinking back at
the college days when I had to get grocery in -20 celcius Canadian winter with
snow coming up to my ankle, I am so happy where I am right now.

## Do I actually need a car?

After finding a real job in Seattle in 2012, the first thing that came to my
mind was thinking about buying a car. I dreaded having to go outside in Waterloo,
Canada during the winter because it was so damn cold. When I think back, I really
could've used a car then. Even a really shitty one would've done the job.
But when I got to Seattle, for a while I was living in Belltown. It's a neighbourhood,
right near by the downtown Seattle where my work was. I could walk for 15 minutes
and I was get to my work. I didn't need a car at all then. I decided against it.

A few months after, I started making some friends in Seattle. A few times I hung
out with them, I really enjoyed meeting them. I wanted to meet them more. But
many of my Seattle friends were living in South Seattle. It's a long way out
from where I am now, and it's a major pain in the neck to take transit there.
So I got around by asking for rides here and there. My friends never minded
giving me a ride, but I felt bad and grateful at the same time for their offer.
I now had a reason I wanted to get a car.

This was a strong reason that pushed me to start the research, but it wasn't big
enough of an incentive to push me to actually act. I would browse Craigslist
and Trucars.com occasionally, but I never really made a real attempt to purchase
a car yet. However, I would learn about another reason why I wanted a car. I went
to [Snowlake](http://www.yelp.com/biz/snow-lake-north-bend).

![](http://i.imgur.com/2tNF1LG.jpg)

It was a beautiful place. I was shocked that I had missed out on such an amazing
sight simply because I didn't have a car to drive there. I swore to myself that
when I get a car, I would drive around the Northwest and enjoy the awesome mountains
and coastlines.

Lastly, I realized I am simply wasting too much time by taking transit. Yes, it
is true that I am saving money, but not driving, but at what cost and the loss
of time? It was a no brainer now. I needed a car.

## What car?

I considered small Asian made cars. Brands like Toyota, Honda, Hyundai, Nissan and Kia
were brands I closely looked at. Compact cars like Corolla, Civic and Elantra
were the car I was interested in the most. I just wanted a car that eats up
little gas and gets me from A to B. Reliability is also important. I needed a car
that will run with just regular maintenance and little to no issues.

I considered both new and used car. I preferred used cars because I was hoping
that I could get a car that's cheaper while giving more value. But as I was
browsing more used cars, it was increasingly clear that Seattle is a terrible
place to buy a used car. Compared to other parts of the country, Seattle used
cars were consistently more expensive. For this reason, I started to check out
new cars.

I went to Honda and Toyota dealers near Seattle downtown area. They were both
helpful but I couldn't help feeling that something felt off. Especially the Honda
dealer was less trust worthy. The day before I wanted to visit the Honda dealer,
I called to make sure that I wanted to have a look at used Kia Forte hatchback.
The call was made at around 8pm. The salesman was saying that I can come and have
a look the next day. When I went early in the morning next day, he told me that
the car was sold right after my call, late at night. I was thinking to myself,
how stupid do they think I am? I was done with dealing with car dealers.

## Buying a used car from a private seller

Although I am writing about this section rather briefly, it was a pretty lengthy
process. It took me around a year to go from researching to purchasing a car mostly
because I felt that I didn't need one immediately. I took time to see what cars of
different year and miles cost how much usually. This helped me greated in determining
what's a good deal and what is no so good. I mistakenly delayed my action in purchasing
a car and missed a few really good deals.
I also got baited by deals that are too good to be true and wasted my time emailing
them.

Last Saturday, I spotted a red 2009 Pontiac Vibe GT on craigslist.
It was auto, only had 86k miles which isn't too bad.
The asking price was $9059. I was interested.
I emailed the seller for a carfax of the vehicle and the seller sent it to me
immediately. This was a good sign. I knew that this seller would be good to
work with.

I called her to see if she'd be available for me to drive the car.
We agreed that I would meet each other in downtown the next day.
I looked around the car and test drove the car in the downtown area.
I tried out steep uphills as well, and Vibe handled it with ease. I didn't
go on a highway which I should've but I wasn't prepared to drive someone else's
car on a highway. At the end of the test run, I asked the seller to arrange
a used car inspection with a mechanic. She agreed to do it.

The 2 days after the test run, we both met again during the mechanic's inspection
of the car. The mechanic also had a Pontiac Vibe serendipitously. Inspection
went smoothly only discovering a cosmetic damage on the car which she fixed later.
After the car was inspected, I agreed to purchased the car and I decided that
I would pay by cashier's cheque. She was fine with that.

The next day, we met at the downtown Seattle DOL office and signed the
title transfer paper, bill of sales and odometer reading paper. The clerk at the
office was very helpful in guiding us through the process. I paid around $950
on sales tax and WA registration fee, and all was good.

Feel free to comment if you have any question purchasing a used car in WA.


Buying a Used Car in Seattle


## tl;dr

- Backbone.js is amazing
- Event driven development was a paradigmatic shift.
- Backbone won't stop you from writing bad code.
- Overwhelmed with other tools
- Creating global variables and methods are not straight forward in Backbone.

Let me preface the entire post by saying, Backbone.js is awesome.
I've been using Backbone.js for last 3 months at work and doing a side project.
I can't imagine pulling them off in pure jQuery alone.
Backbone.js was instrumental in my work.
In this post, I am writing briefly about how I felt about using Backone.js and
some observations I made about the library.

Coming from Rails world with some jQuery flavour in the front end, events
were not a huge deal for me. I never really worried about events in the Rails
side. Even on the front end, I used events in jQuery very limitedly.

On the other hand in Backbone, events are extremely important and useful.
It helps you to write simpler and modular code. Less if-else spaghettis and
chained up functions calling some other functions and so on. With events, even
if one thing falls down, the other part that depends on the event still won't
fall apart.

At the same time though, you can still create a mess when events are chain up
together. I find the debugging sometimes is harder with event driven code
because the links between different parts of the code are not explicit anymore.
A new coder diving into the pre-existing backbone.js codebase will have
some trouble understanding how different parts of the code are working together.
This is because there's less if-else, and explicit chains of functions to
indicate how different codes are working together.

Backbone.js does help you to writing cleaner code. Having more structured js code
with models, views and routes definitely make the code more understandable.
But it seems as though, there's no perfect guard against shitty code.
I still saw myself writing junk and memory hogging code. Rendering a page
multiple times needlessly and making unnecessary ajax calls to server slowing
the page down. So you still to me mind the performance can be hit using
backbone.js.

Backbone.js often comes with many other tools to learn about.

- require.js for loading modules
- marionette.js for giving more structure and proving commonly used functions in backbone
- handlebars.js for templating
- underscore.js for useful functions on arrays, collections, and misc. js objects
- jquery for dom manipulation and ajax

I also started using:

- yeoman for setting up app dev environment
- grunt.js for automating repetitive tasks
- npm and bower for managing js packages

I noticed that doing something I would consider to be really easy to do in
Rails were more complicated and not straight forward. Stuff like global
variables, view helpers, partial views etc were all doable in backbone, but
there's always more than one way to do things in backbone. That makes things hard for
newcomers.

Overall, my adventure in backbone.js is turning out to be a very fruitful one.
I am learning a lot fast and I can't wait to dig deeper into backone and maybe
try out other frameworks like Angular and Ember more seriously.


Random Thoughts On Using Backbone.js For 3 Months


## tl;dr

To use [Twitter Typeahead.js](https://github.com/twitter/typeahead.js/) with Backbone.js,
you need to set up the typeahead function in a Backbone view's `render()`.
And you need to style the typeahead dropdown appropriately with css.

## What is Twitter Typeahead?

![](http://i.imgur.com/AjpDzWL.png)

Twitter Typeahead is a autocomplete/suggestion engine
made for text fields on websites. As a dependency,
Typeahead requires jQuery.

## Typeahead with Backbone

### 1

Check the [Typeahead examples](https://twitter.github.io/typeahead.js/examples/)
and find the example that is suitable for
your app's use case.

### 2

Copy the example js code and create a function
in the view which is responsible for generating the form.

{% highlight js %}
// ...

this.twitterTypeahead: function() {
// constructs the suggestion engine
var states = new Bloodhound({
datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
queryTokenizer: Bloodhound.tokenizers.whitespace,
// `states` is an array of state names defined in "The Basics"
local: \$.map(states, function(state) { return { value: state }; })
});

// kicks off the loading/processing of `local` and `prefetch`
states.initialize();

\$('#bloodhound .typeahead').typeahead({
hint: true,
highlight: true,
minLength: 1
},
{
name: 'states',
displayKey: 'value',
// `ttAdapter` wraps the suggestion engine in an adapter that
// is compatible with the typeahead jQuery plugin
source: states.ttAdapter()
});
},

// ...
{% endhighlight %}

### 3

This function should run within the view's `render()`
function.

{% highlight js %}
// ...
this.redner: function() {
// ...
this.twitterTypeahead();
// ...
},
// ...
{% endhighlight %}

### 4

You need to style the typeahead drowpdown. See
this [stackoverflow thread](http://stackoverflow.com/questions/20198247/twitters-typeahead-js-suggestions-are-not-styled-have-no-border-transparent-b)
for more info.

### 5 Optional

If you happen to use a remote source for data,
use the remote example in the Twitter Typeahead example
page. Also on the server side, set up routes, controller etc.
for allowing the Typeahed to make calls to your server
for data.


High level steps to setting up Twitter Typeahead.js to work with Backbone.js


## tl;dr

I am recounting of how I spent time before I had a computer.

## Before computers

Long before I had a computer, I had several types of toys I gravitated to.
Robots figures! Damn, those were cool. I didn't like them for a long time
though. Far too often, different parts of the robot will break and I
would soon get bored of them. I also like toy cars a lot. One time when my dad
went to LA for work, he brought home [Hot Wheals](http://en.wikipedia.org/wiki/Hot_Wheels).
I remember making rambunctious engine noises and racing the cars with my hand.
They were really fun, but I lost them easily. These small toy cars were
incredibly susceptible to going under different furnitures around the house
and I lost some of them one by one.

There were several others I liked, but there were mostly fads for me.
But there was one type of toy that stuck with me for years.

## Lego

I played with Legos for years. When I think about it, I think legos were
something I played with for the longest time besides computers. I don't think
I really thought about why I liked Legos so much when I was playing with them
at that age. However, it's clear to me that I really enjoyed building things
in retrospect. It could've been that Legos were one thing I had some control over
in my life. Remember, this was when I was told when I to go to bed, what I had to eat,
when I had to wash myself and do all other trivial shit.

Lego was the god-mode sandbox. I spent a lot of time making cool stuff, ugly stuff,
weird stuff and all kinds of random things. One thing I didn't do with lego
was following the instruction. I never bothered with them. Not only was I too
impatient to follow them (if I was raised in US, I was probably diagnosed as ADD),
I found no fun in following the instruction. Sure, it looked impressive to
other friends, but what fun is it if I had to keep the same structure the whole
time without being able to build something else?

Legos were really really big in Korea during 90s. I remember one year, when half
the boys in my class had the same kit called, [Lion Castle, in Korean, 사자성](http://pockgoon.egloos.com/3701611).
If I remember correctly, I think this kit was easily over 100,000 won in Korea, which
is roughly \$100 in US, so it is not a small purchase, especially for Koreans
in 90s.
One of my best friends then had also bought it. I didn't. I remember
begging my parents for it, but one thing I remember is that they never
gave into me begging. They didn't budge an inch. This was the most impressive
Lego kit I've seen until then. For the first time, I wanted to follow the
instruction to build it.

I pretty much stopped playing with Legos abruptly when I moved to Hong Kong.
The delivery company DHL (I still remember!) lost a box that contained my lego.
That was it with my legos. I longed for legos still, but that quickly faded
when I was introduced to a game called Starcraft by my Kumon math tutor. But
I'll save that for later.

## Playground

Korea was a weird place to be during the 90s. Not everyone were well off, but
the kind of poverty people experience just a few decades ago was almost
completely gone. There was a huge surge of optimism that things will get better
and progress was simply inevitable. Of course, I am speaking of pre-late 90s
Asian economic collapse here. Things got dark quickly, but again, I'll leave this
bit for later.

I don't know about Korea too much now, but apparently, there aren't kids in playgrounds
anymore. Kids in general are less common nowadays in Korea. Korea now has the
[5th lowest fertility rate in the world](https://www.cia.gov/library/publications/the-world-factbook/rankorder/2127rank.html).
Kids are probably stuck at home on computers these days.

90s probably marked the last generation of kids playing in playground.
So many kids knew each other just being around the playground.
In most cases, kids got along well. There were some bullies, but they were rare.
We played soccer in small fields and jump off tall slides and swings in mid air.
If I tried any of those now, I'll probably break a few bones here and there.

I will dearly miss the _good ol' days_.


What I was doing before computers


## tl;dr

Wrote a linked list queue in Swift

---

After writing a [stack in Swift](http://serv.github.io/blog/2014/06/06/stack-in-swift/),
I decided that I try writing a queue in Swift.

Again, this tutorial requires you to have
XCode 6 Beta in order to run Swift.

### 1. Define classes

{% highlight swift %}
import Foundation

class Queue {

}
{% endhighlight %}

{% highlight swift %}
import Foundation

class Node {

}
{% endhighlight %}

I created two classes in different files in XCode.

### 2. Attributes and Constructors for Node class

{% highlight swift %}
class Node<T:NSObject> {
    var value: T? = nil
    var next: Node<T>? = nil

    init() {
    }

    init(value: T) {
        self.value = value
    }
}
{% endhighlight %}

`Node` has two attributes: `value` and `next`.
`value` has an optional generic type `T` which may have `nil` value.
`next` has an optional Node<T> type which also my have `nil` value.

`Node` has two `init` constructors: one that takes `value` as an
argument, and another one that doesn't take any argument.

### 3. Attributes and Constructor for Queue class

{% highlight swift %}
import Foundation

class Queue<T:NSObject> {
    var count: Int = 0
    var head: Node<T> = Node<T>()
    var tail: Node<T> = Node<T>()

    init() {
    }
}
{% endhighlight %}

`Queue` class has three attributes `count`, `head` and `tail`.
`head` is the beginning of the queue and `tail` is the end of the queue.

### 4. `isEmpty` function

{% highlight swift %}
import Foundation

class Queue<T:NSObject> {
    var count: Int = 0
    var head: Node<T> = Node<T>()
    var tail: Node<T> = Node<T>()

    init() {
    }

    func isEmpty() -> Bool {
        return self.count == 0
    }
}
{% endhighlight %}

`isEmpty` is simply comparing `self.count == 0`.
If the queue is empty, returns true, if not, false.

### 5. `enqueue` function

{% highlight swift %}
import Foundation

class Queue<T:NSObject> {
    var count: Int = 0
    var head: Node<T> = Node<T>()
    var tail: Node<T> = Node<T>()

    init() {
    }

    func isEmpty() -> Bool {
        return self.count == 0
    }

    func enqueue(value: T) {
        var node = Node<T>(value: value)
        if self.isEmpty() {
            self.head = node
            self.tail = node
        } else {
            node.next = self.head
            self.head = node
        }
        self.count++
    }
}
{% endhighlight %}

`enqueue` function takes a value and creates a node out of it.
It first checks if the queue is empty.
If it is empty, queue's head and tail are set to the node.
If not, the node is inserted in front of the current head of the queue.
Then increment `count`.

### 6. `dequeue` function

{% highlight swift %}
import Foundation

class Queue<T:NSObject> {
    var count: Int = 0
    var head: Node<T> = Node<T>()
    var tail: Node<T> = Node<T>()

    init() {
    }

    func isEmpty() -> Bool {
        return self.count == 0
    }

    func enqueue(value: T) {
        var node = Node<T>(value: value)
        if self.isEmpty() {
            self.head = node
            self.tail = node
        } else {
            node.next = self.head
            self.head = node
        }
        self.count++
    }

    func dequeue() -> T? {
        if self.isEmpty() {
            return nil
        } else if self.count == 1 {
            var temp: Node<T> = self.tail
            self.count--
            return temp.value
        } else {
            var temp: Node<T> = self.tail

            // ??????

            self.count--
            return temp.value
        }
    }
}
{% endhighlight %}

`dequeue` function has 3 cases to handle.

1. When the queue is empty
2. When the queue has one node
3. When the queue has more than one node

In case 1, you just return `nil`.

In case 2, you return the queue's tail node and decrement.

In case 3, you want to return the queue's tail node as well.
But the problem is that we need to know the reference to the node right before
the tail node. **We need `prev` attribute for nodes.**

### 7. Adding `prev` attribute to Node class

{% highlight swift %}
import Foundation

class Node<T:NSObject> {
    var value: T? = nil
    var next: Node<T>? = nil
    var prev: Node<T>? = nil

    init() {
    }

    init(value: T) {
        self.value = value
    }
}
{% endhighlight %}

`prev` attribute is written the same way as `next`.

{% highlight swift %}
import Foundation

class Queue<T:NSObject> {
    var count: Int = 0
    var head: Node<T> = Node<T>()
    var tail: Node<T> = Node<T>()

    init() {
    }

    func isEmpty() -> Bool {
        return self.count == 0
    }

    func enqueue(value: T) {
        var node = Node<T>(value: value)
        if self.isEmpty() {
            self.head = node
            self.tail = node
        } else {
            node.next = self.head
            self.head.prev = node
            self.head = node
        }
        self.count++
    }

    func dequeue() -> T? {
        if self.isEmpty() {
            return nil
        } else if self.count == 1 {
            var temp: Node<T> = self.tail
            self.count--
            return temp.value
        } else {
            var temp: Node<T> = self.tail
            self.tail = self.tail.prev!
            self.count--
            return temp.value
        }
    }
}
{% endhighlight %}

We add `self.tail = self.tail.prev!`, which
sets the new tail node, when the current tail
node is removed.

---

You can see the [full source code on github](https://github.com/serv/algorithms-in-swift/tree/master/DataStructuresAndAlgorithmsInSwift/QueueLinkedList/QueueLinkedList).


Simple Linked List Queue in Swift


##tl;dr
I wrote a stack using linked list in Swift

---

When I try to learn a new language, I try writing
a stack using a linked list. This makes you learn a
few basic things about the language and helps you
become familiar with the language faster. I decided to do
the same with Swift.

As a prerequisite, you need XCode 6 Beta in order to run
Swift.

### 1. Define classes

{% highlight swift %}
import Foundation

class Stack {

}
{% endhighlight %}

{% highlight swift %}
import Foundation

class node {

}
{% endhighlight %}

I created two classes in different files in XCode.

### 2. Define attributes of the Node class

{% highlight swift %}
import Foundation

class node {
  var value: NSObject?
  var next: Node?
}
{% endhighlight %}

I defined two attributes called:

1. `value`
2. `next`

`value` is an optional attribute with `NSObject` type.
First, `value` is optional because it may have `nil` value.
Second, `value` is typed as `NSObject` because we want
the Node to handle multiple types of value such as
integer, double and string.

`next` is also on optional attribute because some nodes may
not have any next node. So `next` might be `nil`. `next`
is typed as `Node` because it points to another node.

### 3. `init` functions in Node

{% highlight swift %}
import Foundation

class Node {
  var value: NSObject?
  var next: Node?

  init() {

  }

  init(value: NSObject) {
    self.value = value
    self.next = Node()
  }
}
{% endhighlight %}

We want two `init` functions because we want to
instantiate a node with some `value` or without anything.
We are done with Node class

### 4. `init` function for Stack and its attributes

{% highlight swift %}
import Foundation

class Stack {
var count: Int = 0
  var head: Node = Node()

  init() {
  }
}
{% endhighlight %}

Stack has `head` and `count` attributes.
`head` points to the top of the stack.
`count` has the total number of nodes in the stack.

### 5. `isEmpty` function

{% highlight swift %}
import Foundation

class Stack {
var count: Int = 0
  var head: Node = Node()

  init() {
  }

  func isEmpty() -> Boolean {
    return self.count == 0
  }
}
{% endhighlight %}

`isEmpty` function returns `true` if the stack is empty
and returns `false` if the stack is not empty.

### 6. `push` function

`push` function inserts some value to the top of the stack.

{% highlight swift %}
func push(value: NSObject) {
  if isEmpty() {
    self.head = Node()
  }

  var node = Node(value: value)
  node.next = self.head
  self.head = node
  self.count++
}
{% endhighlight %}

### 7. `pop` function

`pop` function removes some value from the top
of the stack.

{% highlight swift %}
func pop() -> NSObject? {
  if isEmpty() {
    return nil
  }

  var node = self.head
  self.head = node.next!
  self.count--

  return node.value
}
{% endhighlight %}

This function has optional return type because if the
stack is empty, it should return nothing, `nil`.

In the line, `self.head = node.next!`, we use `!` to force
the optional variable to unwrap its value.

---

You can see the [full source code on github](https://github.com/serv/algorithms-in-swift/tree/master/DataStructuresAndAlgorithmsInSwift/StackLinkedList/StackLinkedList).


Stack in Swift


##tl;dr

I worked on chat/photo sharing app called [Bubble](http://app-bubble.com)
with 4 others. We won the runner-up for
judge's choice award in [University of Washington's
HCDE Capstone & Research Showcase].

![](http://i.imgur.com/Rre9eGu.jpg)

---

I worked with 4 masters students from UW's HCD program to build a photo sharing
chat app with a _shocking_ twist. The twist is that the app does not need
internet connection to work. The app is called [Bubble](http://hcdeevee.tumblr.com/).

We started working together in January 2014. We began
with brainstorming what kind of thing we wanted
to work with. We literally could not gauge what each
of us were interested in. So we threw together a dozen
something ideas and we narrowed down to a vague
location check-in app. The idea didn't stop evolving there. We continued to carve
and morph the idea. Eventually the app turned out to be
something completely unexpected.

Bubble is a chat/photo sharing app that doesn't
rely on internet connection.

Back in 2009 when I used to commute to my work to Merrill Lynch
in Toronto, I felt incredibly bored in the subway.
I had a old school flip phone, so I couldn't do much,
but I had unlimited text and I would text a lot.
But in the subway without any connection, I was left
doing nothing. There and then, I was thinking to myself
how awesome it would be to still be able to chat
with others without having signal. I thought of
some sort of mesh network then. Decentralized, ad-hoc
networks that could be created anywhere. I also
imagined a world where companies like Rogers and Bell
were irrelevant forces in telecommunication. So this
idea of complete decentralized network captured
my wild imaginations.

Fast forward to 2014, I learned that iOS has the
precise capability to do this. I didn't really let
my team know, but it was an amazing feeling. I was
finally working on something that I only dreamed of
only a few years ago. And this app I was working on
wasn't just for chat, it could also send photos. The
idea was infatuating. The app was gold.

I will continue to work on the app to see it in the AppStore.
I can't wait for that moment to come.

![](http://i.imgur.com/ADoh5P6.jpg)


Making Bubble


I come across websites that break the Back button too often for bad reasons.
Back button is one of the longest living orthodoxies in web people take for
granted, and rightly so.

![back](http://i.imgur.com/3FAYPf6.png?1)

Back button is useful. It serves its purpose for navigation so perfectly that
there's been little to no change done it for decades. Phones subsequently
adopted it without modifying what it does that much. Back button remains
useful in mobile browser even more so because other kinds of precise actions
require so much more effort to pull off. Most users resort to the back button
for most of rudimentary navigations.

Back button gets broken with using `window.location` which will redirect the page to the address that `window.location` set to.

## Anatomy of Broke Back Button

![location](http://i.imgur.com/KBSu4K3.jpg)

1. User is on page A.
2. User clicks on a link and goes to B.
3. B has `window.location` set to page C.
4. User is finished with the page C, and wants to return to A.
5. User clicks back.
6. User is back to B, and is immediately redirected to C again.

There are some ways to mitigate the problem.

- Use redirect with countdown so that the user can get a chance to click
  back button again, while the countdown happens.
- If the redirect is to a mobile site for mobile users, try building
  adaptive or responsive site that doesn't require rediecting the user to a
  new site
- Use a gentle call to action that prompts user to the mobile, instead of
  a silent redirect.

Not breaking the default behaviour of the back button is a general rule of
thumb. I am sure there are times, when you must end up breaking the back
button for a bigger gain. :)


Try Not To Break The Back Button


##tl;dr

- Mobile web sucks and mobile native apps are way better.
- HTML5 is not a Messiah.
- no one is really interested in making mobile web better.

Imagine that we are in 2020, 6 years away from now.

We wouldn't have quite gone boldly where no man had gone before in our
universe, but we would have at least explored mobile experience tirelessly
by then. Thousands of photos apps would have been made. WhatsApp copies
would have iterated upon iterations. First there was Angry Bird, then
Flappy Bird, then some more Angry Flappy Birds. Dead horses would be beaten
more than one can count. So much would have changed and improved, but one
thing still remains the same. Mobile web experience will likely still lag behind the native app experience.

![html5-op](http://i.imgur.com/91gZqO4.png)

Inability of browsers to take advantage of phone hardware has been stated
as a reason why mobile web is lagging behind the native app. But I cast
my skepticism cautiously.

[Changes in HTML5](https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5)
in recent years have been absolutely mind blowing.
You can replicate native
app experience on browser (though it is not easy). HTML5 in many ways have
caught up to where native apps are right now. Optimization on browser for
mobile still remains as a difficult task, but so is developing for
native app. So what is dragging mobile web experience down?

## A Vicious Cycle

![cycle](http://i.imgur.com/GM4tFzI.jpg)

Mobile web development process is in a gridlock that continues to worsen
mobile web. The incentives for users, management and developers are all
uniformly against mobile web. Users have a preconceived notion that mobile
web sucks, and this belief is justified. It sucked for a long time, and even
today, it's not that great. Software companies are aware of this consensus
and devote less time and developer resources into making mobile web
experience better. Mobile space for native apps, they say.
Web developers return the favor by creating a web app
that barely meets the minimum expectations set by the management. I want
web to win, but I am not optimistic about it.

Mobile web is suffering from a great stagnation. Effectively distributing
mobile web app to users are practical non-existent. Lack of standardized
monetization scheme hinders any development effort in mobile web. Much of
the web still remains apathetic to mobile users. Discovery process of mobile
web? What discovery process? And mobile web still feels clunky, slow and
off-beat. Mobile web experience is years behind native app experience. In
the age when a year represents an eternity of changes and improvements,
mobile web will not be able to attract users from native app for another few
years.


Unfortunate State of Mobile Web


I am *anti-Eclipse* just because, so I like to add `.jar` file manually.

1. Create `~/.dev` directory
2. Add the jar file (in my case, `junit-4.11.jar`) you want to include in `~/.dev`.
3. In `~/.bash_profile`, add `export CLASSPATH=$CLASSPATH:~/.dev/junit-4.11.jar`
4. Close the current terminal and open a new terminal.

My system is Mac OS X Mavericks version 10.9.1.


Adding a .jar file on test classpath


## tl;dr

- What is Quick Find
- Programming Quick Find in Java
- What is Quick Union
- Programming Quick Union in Java

![algo1](http://i.imgur.com/nlWL17x.png)

Let me give a big shout out to [Coursera's Algorithm I course](https://class.coursera.org/algs4partI-004/) currently running right now. I went through week 1 lecture material that covered, topics in union find and analyzing algorithm. In this blog post, I will focus on the first part, union find.

## Union Find

Say there are N nodes in a given set. What kind of data
structure would it take to represent that these nodes are
connected to one another? What kind of algorithm would we need
to use to connect one node to another? What do we do to check
if two nodes are connect? These questions will be answered
through learning about quick find and quick union
implementations of union find.

## Quick Find

Quick find is an algorithm that places emphasis on finding out
if two nodes are connected or not. We can use a typical array to
represent a set of nodes and their connections to each other.

{% highlight java %}
{1,2,3,4,5};
{% endhighlight %}

Indexes of the array are the nodes and the values of the array
are the connections. When the value of one index equals that of
another node's value, then the two nodes are connected.

Quick Find approach has a very fast method to check connectivity.
To check the connectivity, you just need to access the array
twice. However, union is extremely slow. Since whenever you do
union, you have to go through all the array, you will end up
with O(N) runtime.

The quick find has 3 methods. They are:

- constructor: prepares the set as an array
- connected: checks if two nodes are connected or not
- union: connects two nodes

{% highlight java %}
// QuickFind.java
public class QuickFind {
  private int[] id;
  private int count;

  // constructor
  // Given N, create an array with N elements
  public QuickFind(int N) {
    count = N;
    id = new int[N];
    for (int i = 0; i < N; i++) {
      id[i] = i;
    }
  }

  // if values of two index are the same,
  // return true, else false
  public boolean connected(int p, int q) {
    return id[p] == id[q];
  }

  // connect two nodes
  public void union(int p, int q) {
    for (int i = 0; i < count; i++) {
      if (id[i] == id[p]) {
        id[i] = id[q];
      }
    }
    System.out.println(p + " " + q);
  }

  public void print() {
    StringBuilder temp = new StringBuilder();
    temp.append("");

    for(int i = 0; i < count; i++) {
      temp.append(id[i]);
      temp.append(" ");
    }
    String strI = temp.toString();
    System.out.println(strI);
  }

  public static void main(String[] args) {
    QuickFind qf = new QuickFind(10);
    qf.union(5,6);
    qf.print();
    qf.union(0,9);
    qf.print();
    qf.union(9,3);
    qf.print();
    qf.union(2,1);
    qf.print();
    qf.union(7,0);
    qf.print();
    qf.union(4,8);
    qf.print();
  }
}
{% endhighlight %}

###To run this java program

1. Compile the program. `javac QuickFind.java`
2. Run the compiled program. `java QuickFind`

## Quick Union

Quick union improves greatly in union operation. Instead of
going through entire array during union, you attach one node's
root to another's root.

{% highlight java %}
// QuickUnion.java
public class QuickUnion {
  private int[] id;
  private int[] sz;
  private int count;

  public QuickUnion(int N) {
    count = N;
    id = new int[N];
    sz = new int[N];
    for (int i = 0; i < N; i++) {
      id[i] = i;
      sz[i] = 0;
    }
  }

  private int root(int i) {
    while (i != id[i]) {
      i = id[i];
    }
    return i;
  }

  public boolean connected(int p, int q) {
    return root(p) == root(q);
  }

  public void union(int p, int q) {
    int i = root(p);
    int j = root(q);
    if (i == j) return;
    if (sz[i] < sz[j]) { id[i] = j; sz[j] += sz[i]; }
    else { id[j] = i; sz[i] += sz[j]; }
  }

  public void print() {
    StringBuilder temp = new StringBuilder();
    temp.append("");

    for(int i = 0; i < count; i++) {
      temp.append(id[i]);
      temp.append(" ");
    }
    String strI = temp.toString();
    System.out.println(strI);
  }

  public static void main(String[] args) {
    int N = StdIn.readInt();
    QuickUnion qu = new QuickUnion(N);
    while (!StdIn.isEmpty()) {
      int p = StdIn.readInt();
      int q = StdIn.readInt();
      if (!qu.connected(p, q)) {
        qu.union(p, q);
        StdOut.println(p + " " + q);
      }
    }
    qu.print();
  }
}
{% endhighlight %}

---
### Edited on Feb 11, 2014

- Code for `QuickUnion.java` and `QuickFind.java` were updated to fix some
bugs


[Coursera Algorithms I: Week 1] Quick Find and Quick Union in Java


## tl;dr

- Hate TDD blah blah
- A directory structure of a Ruby program
- Setting up RSpec
- Setting up FactoryGirl

> Setting up RSpec and FactoryGirl wanted me to choke
> a few people.
>
> -- *John Smith*, renown Rubyist,
> husband, author of no books, father of none

#### An Adaquate Response by @tenderlove

<blockquote class="twitter-tweet" lang="en"><p>Here is a Friday Hug. Deal with it. (HAPPY FRIDAY!!! &lt;3&lt;3&lt;3) <a href="http://t.co/6i5rNWqybv">pic.twitter.com/6i5rNWqybv</a></p>&mdash; Aaron Patterson (@tenderlove) <a href="https://twitter.com/tenderlove/statuses/426779493747810305">January 24, 2014</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>

O boy, do I hate test driven development. Nothing feels
worse than writing clean automated tests only to realize that
they've just rendered useless by a small change in an
application feature.
And yet, we writes tests anyway because they pay off in
the long run.

With that in mind, I wanted to write a guide on preparing RSpec
and FactoryGirl for writing Ruby programs. RSpec and FactoryGirl
are great Gems that help you immensely in writing automated
tests, but I've had times when I would curse gods in just
prepping RSpec and FactoryGirl.

## A Directory Structure of a Ruby Program

There is no *one right way* to structure a Ruby program.
However, from my experience, I notice myself increasingly
structuring Ruby programs as if I am writing a Ruby Gem.
Others looking at the code understand the program better,
if they are familiar with how Ruby Gems are typically written.
Modularized code helped me debug easier as well.

Let's say we are writing a Ruby program for a linked list with
a List class and a Node class. Then we will have a directory
structure like below.

	├── Gemfile
	├── Gemfile.lock
	├── lib
	│   └── models
	│       ├── list.rb
	│       └── node.rb
	└── spec
	    ├── factories
	    │   ├── lists.rb
	    │   └── nodes.rb
	    ├── models
	    │   ├── list_spec.rb
	    │   └── node_spec.rb
	    └── spec_helper.rb

- `Gemfile`: contains a list of Gems used in the program
- `lib`: a directory that contains code that runs the program
- `spec`: a directory that contains RSpec and FactoryGirl
related code
- `spec/factories`: a directory that contains factories for
FactoryGirl

## Setting Up RSpec and FactoryGirl

#### Tutorial Environment

- Mac OS X Mavericks Version 10.9.1
- ruby 2.1.0p0 (2013-12-25 revision 44422) [x86_64-darwin12.0]
- rspec 2.14.7

### 1. Gemfile

Prepare Gemfile with RSpec and FactoryGirl.

{% highlight ruby %}
# Gemfile
source 'https://rubygems.org'

gem "rspec"
gem "factory_girl"
{% endhighlight %}

Run `$ bundle install`.

### 2. Prepare RSpec

RSpec is a Ruby Gem that helps you write automated tests.

{% highlight ruby %}
# spec/spec_helper.rb
require 'rubygems'

RSpec.configure do |config|
end
{% endhighlight %}

### 3. Prepare FactoryGirl

FactoryGirl is a Ruby Gem that mocks certain classes for
automated tests. We will use FactoryGirl to mock nodes and
lists.

Add this to the RSpec config.

{% highlight ruby %}
# spec/spec_helper.rb
require 'rubygems'
require 'factory_girl'

RSpec.configure do |config|
  config.include FactoryGirl::Syntax::Methods
end
{% endhighlight %}

### 4. Building Factories

These are just empty factories.

{% highlight ruby %}
# spec/factories/lists.rb
FactoryGirl.define do
  factory :list do
  end
end
{% endhighlight %}

{% highlight ruby %}
# spec/factories/nodes.rb
FactoryGirl.define do
  factory :node do
  end
end
{% endhighlight %}

### 5. Describing Models in RSpec

And these are just empty model specs.

{% highlight ruby %}
# spec/models/node_spec.rb
require 'spec_helper'

describe Node do

end
{% endhighlight %}

{% highlight ruby %}
# spec/models/list_spec.rb
require 'spec_helper'

describe List do

end
{% endhighlight %}

Try running RSpec tests now. Run `$ rspec spec`.

You should get an error message that reads something like:

	/...../spec/models/list_spec.rb:3:in `<top (required)>': uninitialized constant List (NameError)

### 6. Add Models

The error above is caused by the fact that we don't actually
have any models yet.

{% highlight ruby %}
# lib/models/node.rb
class Node
end
{% endhighlight %}

{% highlight ruby %}
# lib/models/list.rb
class List
end
{% endhighlight %}

Try running RSpec tests again. Run `$ rspec spec`.

You should get the same error message that reads:

	/...../spec/models/list_spec.rb:3:in `<top (required)>': uninitialized constant List (NameError)

### 7. Adding Dependancies in RSpec Config

We are getting this error because RSpec is unable to location the models we just created.

{% highlight ruby %}
# spec/spec_helper
require 'rubygems'
require 'factory_girl'

require_relative '../lib/models/list'
require_relative '../lib/models/node'

RSpec.configure do |config|
  config.include FactoryGirl::Syntax::Methods
end
{% endhighlight %}

Now try running `$ rspec spec`.

Voila! Tests are running! Well, we don't really have any tests,
but we have RSpec finally running.

	No examples found.


	Finished in 0.00006 seconds
	0 examples, 0 failures

### 8. Running Tests

We will write very basic test that confirms that we can create
a Node object and a List object.

{% highlight ruby %}
# spec/models/node_spec.rb
require 'spec_helper'

describe Node do  
  describe 'instantiation' do
    let!(:node) { build(:node) }

    it 'instantiates a list' do
      expect(node.class.name).to eq("Node")
    end
  end
end
{% endhighlight %}

{% highlight ruby %}
# spec/models/list_spec.rb
require 'spec_helper'

describe List do  
  describe 'instantiation' do
    let!(:list) { build(:list) }

    it 'instantiates a list' do
      expect(list.class.name).to eq("List")
    end
  end
end
{% endhighlight %}

Try running the tests `$ rspec spec`.

You should get an error saying that node and list factories
have not been registered.

	FF

	Failures:

	  1) List instantiation instantiates a list
	     Failure/Error: let!(:list) { build(:list) }
	     ArgumentError:
	       Factory not registered: list
	     # ./spec/models/list_spec.rb:5:in `block (3 levels) in <top (required)>'

	  2) Node instantiation instantiates a list
	     Failure/Error: let!(:node) { build(:node) }
	     ArgumentError:
	       Factory not registered: node
	     # ./spec/models/node_spec.rb:5:in `block (3 levels) in <top (required)>'

	Finished in 0.00074 seconds
	2 examples, 2 failures

	Failed examples:

	rspec ./spec/models/list_spec.rb:7 # List instantiation instantiates a list
	rspec ./spec/models/node_spec.rb:7 # Node instantiation instantiates a list

The problem is caused by RSpec not being able to find
FactoryGirl files.

Add FactoryGirl files as dependancies running RSpec.

{% highlight ruby %}
# spec/spec_helper.rb
require 'rubygems'
require 'factory_girl'

require_relative '../lib/models/list'
require_relative '../lib/models/node'
require_relative '../spec/factories/lists.rb'
require_relative '../spec/factories/nodes.rb'

RSpec.configure do |config|
  config.include FactoryGirl::Syntax::Methods
end
{% endhighlight %}

Now you should be able to run tests successfully.

	..

	Finished in 0.00151 seconds
	2 examples, 0 failures


Basic Testing Setup with RSpec and FactoryGirl


## tl;dr

- Basic components and methods of a linked list
- Testing singly linked list implementation
- Programming a singly linked list using Ruby
- Complete code can be view on [github](https://github.com/serv/learning/tree/master/serv.github.io/algorithms/singly_linked_list)

## Components

A linked list is a data structure that consists of nodes that
make a sequence. A typical linked list consists of these following components.

- Head pointer: a node that indicates the start of the list
- Tail pointer: a node that indicates the end of the node
- Head node: a node that the head pointer points to
- Tail node: a node that the tail node points to
- Pointer: an attribute of node that _links_ to another node
- Linked list: a sequence of nodes interconnected by pointers
- Singly linked list: a sequence of nodes connected by pointers
  going from a previous node to its next node

![linked list components](http://i.imgur.com/V5dj7GG.jpg)
Figure 1: Components of a singly linked list

There are many ways to implement a linked list. You will easily
find other ways to create a head and a tail for a linked list
in other tutorials. Other tutorials might have different methods
for their linked lists.

## Methods

Here are some common methods for a singly linked list and their
average big-O notations. I gave intuitive reasons for given
runtimes.

- Searching [O(n)]: You typically have to go through a certain
  number of nodes in order to find the node you are looking for
  in a linked list. In the best case scenario, you will find the
  node you are looking for at the head node and the runtime is
  O(1). In the worst case scenario, you will find the node you are
  looking for right before the tail node and the runtime is O(n).
- Push head [O(1)]: You can immediately access the head node's
  location. Placing a new node in front of the head node is
  trivial and it will have a constant runtime.
- Push tail [O(1)]: You can immediately access the tail node's
  location. Placing a new node in front of the head node is
  trivial and it will have a constant runtime.
- Pop head [O(1)]: You can immediately access the head node's
  location. Deleting the head node is trivial and it will have a
  constant runtime.
- Pop tail [O(n)]: In a singly linked list, the runtime for popping ting is O(n) surprisingly. You can immediately access the head node's
  location. However, setting the second last node to be the new head involves intreating through all nodes in the list.
- Print [O(n)]: You have to go through all nodes in the list to
  get the values of each nodes.
- Reverse [O(n)]: You have to go through all the nodes in the
  list to reverse the pointers.

## Implementation in Ruby

We will create a Node class and a List class. Before we do that,
let's create some tests for the implementation.

### Prepare RSpec & FactoryGirl

Please follow [Basic Testing Setup with RSpec and FactoryGirl](http://serv.github.io/blog/2014/02/04/basic-testing-setup-with-rspec-and-factorygirl/) to prepare RSpec and FactoryGirl for testing.

### Planning Models

We need to plan attributes of the Node model and the List
model.

- Node
  - value
  - point
- List
  - head
  - tail
  - count

### Building Factories

Now that we know what attributes are needed for models, we can
build factories.

{% highlight ruby %}

# spec/factories/lists.rb

FactoryGirl.define do
factory :list do
head { head }
tail { tail }
count { count }
end
end
{% endhighlight %}

{% highlight ruby %}

# spec/factories/nodes.rb

FactoryGirl.define do
factory :node do
value { value }
point { point }
end
end
{% endhighlight %}

Running the tests now should result in failure.

### Instantiation tests

Let's write some tests for building list and node instantiations.

{% highlight ruby %}

# spec/models/list_spec.rb

require 'spec_helper'

describe List do
describe 'instantiation' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'instantiates a list' do
      expect(list.class.name).to eq("List")
    end

end

describe '#new' do
let!(:list) { List.new }

    it 'build a list' do
      expect(list.head).to eq(nil)
      expect(list.tail).to eq(nil)
      expect(list.count).to eq(0)
    end

end
end
{% endhighlight %}

{% highlight ruby %}

# spec/models/node_spec.rb

require 'spec_helper'

describe Node do
describe 'instantiation' do
let!(:node) { build(:node, value: 1, point: nil) }

    it 'instantiates a list' do
      expect(node.class.name).to eq("Node")
    end

end

describe '#new' do
let!(:node) { Node.new }

    it 'build a list' do
      expect(node.value).to eq(nil)
      expect(node.point).to eq(nil)
    end

end
end
{% endhighlight %}

If you run the tests now, they should still fail. This is because we
haven't written `new` methods for the models yet.

### `new` Method and attributes

Now we will write new methods and attributes for the models.

{% highlight ruby %}

# lib/models/node.rb

class Node
attr_accessor :value, :point

def initialize
@value = nil
@point = nil
end
end
{% endhighlight %}

{% highlight ruby %}

# lib/models/list.rb

class List
attr_accessor :head, :tail, :count

def initialize
@head = nil
@tail = nil
@count = 0
end
end
{% endhighlight %}

### All Methods with Tests

{% highlight ruby %}

# spec/models/list_spec.rb

require 'spec_helper'

describe List do
describe 'instantiation' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'instantiates a list' do
      expect(list.class.name).to eq("List")
    end

end

describe '#new' do
let!(:list) { List.new }

    it 'build a list' do
      expect(list.head).to eq(nil)
      expect(list.tail).to eq(nil)
      expect(list.count).to eq(0)
    end

end

describe '#push_head(value)' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'no node in the list' do
      random_value = rand(10)
      list.push_head(random_value)
      list.head.value.should == random_value
    end

    it 'one node in the list already' do
      random_value_1 = rand(10)
      random_value_2 = rand(10)
      list.push_head(random_value_1)
      list.push_head(random_value_2)
      list.head.value.should == random_value_2
      list.tail.value.should == random_value_1
    end

    it 'many nodes in the list already' do
      tail = rand(10)
      head = rand(10)
      list.push_head(tail)
      list.push_head(rand(10))
      list.push_head(rand(10))
      list.push_head(head)
      list.head.value.should == head
      list.tail.value.should == tail
    end

    it 'increments count' do
      list.count.should == 0
      list.push_head(rand(10))
      list.push_head(rand(10))
      list.count.should == 2
      list.push_head(rand(10))
      list.count.should == 3
    end

end

describe '#push_tail(value)' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'no node in the list' do
      random_value = rand(10)
      list.push_tail(random_value)
      list.head.value.should == random_value
      list.tail.value.should == random_value
    end

    it 'one node in the list already' do
      random_value_1 = rand(10)
      random_value_2 = rand(10)
      list.push_tail(random_value_1)
      list.push_tail(random_value_2)
      list.head.value.should == random_value_1
      list.tail.value.should == random_value_2
    end

    it 'many nodes in the list already' do
      tail = rand(10)
      head = rand(10)
      list.push_tail(head)
      list.push_tail(rand(10))
      list.push_tail(rand(10))
      list.push_tail(tail)
      list.head.value.should == head
      list.tail.value.should == tail
    end

    it 'increments count' do
      list.count.should == 0
      list.push_tail(rand(10))
      list.push_tail(rand(10))
      list.count.should == 2
      list.push_tail(rand(10))
      list.count.should == 3
    end

end

describe '#pop_head' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'no node in the list' do
      list.pop_head.should == false
    end

    it 'one node in the list already' do
      random_value_1 = rand(10)
      list.push_tail(random_value_1)
      list.head.value.should == random_value_1
      list.tail.value.should == random_value_1
      list.pop_head.should == random_value_1
      list.count.should == 0
    end

    it 'many nodes in the list already' do
      tail = rand(10)
      head = rand(10)
      list.push_tail(head)
      list.push_tail(tail)
      list.head.value.should == head
      list.tail.value.should == tail

      list.pop_head.should == head
      list.count.should == 1
      list.head.value.should == tail
      list.tail.value.should == tail
    end

end

describe '#pop_tail' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'no node in the list' do
      list.pop_tail.should == false
    end

    it 'one node in the list already' do
      random_value_1 = rand(10)
      list.push_tail(random_value_1)
      list.head.value.should == random_value_1
      list.tail.value.should == random_value_1
      list.pop_tail.should == random_value_1
      list.count.should == 0
    end

    it 'many nodes in the list already' do
      tail = 1
      head = 2
      list.push_head(head)
      list.push_tail(tail)
      list.head.value.should == head
      list.tail.value.should == tail

      list.pop_tail.should == tail
      list.count.should == 1
      list.head.value.should == head
      list.tail.value.should == tail
    end

end

describe '#nodes' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'prints lists' do
      nodes = []

      5.times do |i|
        x = rand(5)
        nodes << x
        list.push_tail(x)
      end

      list.nodes.should == nodes
    end

end

describe '#reverse' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'reverses list with many nodes' do
      nodes = [1,2,3,4,5]

      nodes.each do |n|
        list.push_tail(n)
      end

      list.reverse.nodes.should == nodes.reverse
    end

    it 'reverses list with one node' do
      nodes = [1]

      nodes.each do |n|
        list.push_tail(n)
      end

      list.reverse.nodes.should == nodes.reverse
    end

    it 'reverses list with no node' do
      nodes = []

      nodes.each do |n|
        list.push_tail(n)
      end

      list.reverse.should == false
    end

end

describe '#search(value)' do
let!(:list) { build(:list, head: nil, tail: nil, count: 0) }

    it 'search a node in a list with no nodes' do
      list.search(1).should == false
    end

    it 'search a node in a list' do
      nodes = [1,2,3,4,5]

      nodes.each do |n|
        list.push_tail(n)
      end

      list.search(4).value.should == 4
      list.search(6).should == false
    end

end
end
{% endhighlight %}

{% highlight ruby %}

# lib/models/list.rb

class List
attr_accessor :head, :tail, :count

def initialize
@head = nil
@tail = nil
@count = 0
end

# Push a tail node

def push_head(value)
node = Node.new
node.value = value

    if @count == 0
      @head = node
      @tail = node
      node.point = nil
    else
      node.point = @head
      @head = node
    end
    @count += 1

end

# Push a head node

def push_tail(value)
node = Node.new
node.value = value

    if @count == 0
      @head = node
      @tail = node
      node.point = nil
    else
      @tail.point = node
      @tail = node
      node.point = nil
    end

    @count += 1

end

# Get head value and delete head node

def pop_head
return false if @count < 1

    pop = @head.value
    @head = @head.point
    @count -= 1
    pop

end

# Get tail value and delete tail node

def pop_tail
return false if @count < 1

    pop = @tail.value
    prev = nil
    current = @head

    while current
      prev = current
      current = current.point
    end

    @tail = prev
    prev.point = nil

    @count -= 1
    pop

end

# Returns all nodes in a list

def nodes
return false if @count < 1

    current = @head
    str = []

    while current
      str << current.value
      current = current.point
    end

    str

end

# Prints all nodes in a list

def print
puts nodes.join('-')
end

# Reverse a list

def reverse
if @count < 1
false
elsif @count == 1
self
else
first = nil
second = @head

      while second
        temp = second.point

        second.point = first
        first = second

        second = temp
      end

      @head.point = nil
      t_head = @head
      @head = @tail
      @tail = t_head

      self
    end

end

# Searches a value in a list

def search(value)
if @count < 1
false
else
current = @head

      while current
        return current if value == current.value
        current = current.point
      end

      false
    end

end
end
{% endhighlight %}
