I added in mouse picking. The bounding boxes need not be axis aligned, although I'm too lazy to rename them from AABBs to just BBs. I ended up breaking the game board into 8x8 cells, each with a bounding box, so triangle-ray intersection is speedy-ish.

The relevant code is:

void get_collision_location(size_t x, size_t y)
{
	// Get intersection point closest to camera

	vec3 ray = screen_coords_to_world_coords(x, y, win_x, win_y);

	float t = 0;

	clicked_col_loc = background;
	bool first_assignment = true;

	mat4 inv = inverse(board_mesh.model_mat);
	vec4 start = inv * vec4(main_camera.eye, 1.0);
	vec4 direction = inv * vec4(ray, 0.0);
	direction = normalize(direction);

	if (true == board_mesh.intersect_AABB(start, direction))
	{
		vec3 closest_intersection_point;

		for (size_t cell_x = 0; cell_x < board_mesh.num_cells_wide; cell_x++)
		{
			for (size_t cell_y = 0; cell_y < board_mesh.num_cells_wide; cell_y++)
			{
				if (true == board_mesh.intersect_triangles(start, direction, closest_intersection_point, cell_x, cell_y))
				{
					closest_intersection_point = board_mesh.model_mat * vec4(closest_intersection_point, 1);

					clicked_collision_location = closest_intersection_point;
					clicked_col_loc = game_board;
					clicked_cell_x = cell_x;
					clicked_cell_y = cell_y;
					first_assignment = false;
				}
			}
		}
	}

	for (size_t i = 0; i < player_game_piece_meshes.size(); i++)
	{
		mat4 inv = inverse(player_game_piece_meshes[i].model_mat);
		vec4 start = inv * vec4(main_camera.eye, 1.0);
		vec4 direction = inv * vec4(ray, 0.0);
		direction = normalize(direction);

		if (true == player_game_piece_meshes[i].intersect_AABB(start, direction))
		{
			vec3 closest_intersection_point;

			if (true == player_game_piece_meshes[i].intersect_triangles(start, direction, closest_intersection_point, false))
			{
				closest_intersection_point = player_game_piece_meshes[i].model_mat * vec4(closest_intersection_point, 1);

				if (first_assignment)
				{
					clicked_collision_location = closest_intersection_point;

					clicked_col_loc = player_game_piece;
					clicked_collision_location_index = i;

					first_assignment = false;
				}
				else
				{
					vec3 c0 = vec3(main_camera.eye) - closest_intersection_point;
					vec3 c1 = vec3(main_camera.eye) - clicked_collision_location;

					if (length(c0) < length(c1))
					{
						clicked_collision_location = closest_intersection_point;

						clicked_col_loc = player_game_piece;
						clicked_collision_location_index = i;
					}
				}
			}
		}
	}

	// Nothing was clicked on, so the background is set
	if (first_assignment)
	{
		clicked_collision_location = vec3(0, 0, 0);
		clicked_col_loc = background;
	}
}

It was @joej who taught me about the model matrix, and how it can be oriented in any direction, because it's basically a orthonormal basis + translation wrapped into one.