osm-parser-aqp-combis/main.go

package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
)

type OsmBounds struct {
	MinLat float64 `xml:"minlat,attr"`
	MinLon float64 `xml:"minlon,attr"`
	MaxLat float64 `xml:"maxlat,attr"`
	MaxLon float64 `xml:"maxlon,attr"`
}

type OsmNode struct {
	Id        int64   `xml:"id,attr"`
	Lat       float64 `xml:"lat,attr"`
	Lon       float64 `xml:"lon,attr"`
	Version   int64   `xml:"version,attr"`
	Timestamp string  `xml:"timestamp,attr"`
	Changeset int64   `xml:"changeset,attr"`
	Uid       int64   `xml:"uid,attr"`
	User      string  `xml:"user,attr"`
}

type OsmMember struct {
	Type string `xml:"type,attr"`
	Ref  int64  `xml:"ref,attr"`
	Role string `xml:"role,attr"`
}

type OsmTag struct {
	K string `xml:"k,attr"`
	V string `xml:"v,attr"`
}

type OsmWay struct {
	Id        int64   `xml:"id,attr"`
	Version   int64   `xml:"version,attr"`
	Timestamp string  `xml:"timestamp,attr"`
	Changeset int64   `xml:"changeset,attr"`
	Uid       int64   `xml:"uid,attr"`
	User      string  `xml:"user,attr"`
	Nds       []OsmNd `xml:"nd"`
}

type OsmNd struct {
	Ref int64 `xml:"ref,attr"`
}

type OsmRelation struct {
	Id        int64       `xml:"id,attr"`
	Version   int32       `xml:"version,attr"`
	Timestamp string      `xml:"timestamp,attr"`
	Changeset int64       `xml:"changeset,attr"`
	Uid       int64       `xml:"uid,attr"`
	User      string      `xml:"user,attr"`
	Members   []OsmMember `xml:"member"`
	Tags      []OsmTag    `xml:"tag"`
}

type OsmDocument struct {
	Note      string        `xml:"note"`
	Bounds    OsmBounds     `xml:"bounds"`
	Nodes     []OsmNode     `xml:"node"`
	Relations []OsmRelation `xml:"relation"`
	Ways      []OsmWay      `xml:"way"`
}

type CombiPair struct {
	Departure *OsmRelation
	Return    *OsmRelation
}

type CombiData struct {
	// Number that succeedes the Company name.
	// E.g.: 7
	Id int
	// Name of the operator of the route.
	// E.g.: "C7 - AqpMasivo"
	Company string
	// Name of the route
	// E.g.: "Combi B: Villa Santa Rosa -> Terminal Terrestre"
	Name string
	// Indentifiable name of the route
	Ref           string
	Color         string
	Members       *[]OsmMember
	ReturnMembers *[]OsmMember
}

// Type to be consumed by the client
// Represents a single company
type CombiLine struct {
	Id       int    `json:"id"`
	Name     string `json:"name"`
	District string `json:"district"`
	Color    string `json:"color"`
}

type CombiRouteContainer struct {
	Routes []CombiRoute
}

type CombiRoute struct {
	Name      string       `json:"name"`
	Departure *[][]float64 `json:"departure"`
	Return    *[][]float64 `json:"return"`
}

func main() {
	log.Println("Begin processing")

	xmlData, err := os.ReadFile("aqp_map.xml")
	if err != nil {
		panic(err)
	}

	var osmDocument OsmDocument
	err = xml.Unmarshal(xmlData, &osmDocument)
	if err != nil {
		panic(err)
	}
	log.Println("XML unmarshal finished")

	// Create a map of IDs to Nodes
	nodesMap := make(map[int64]*OsmNode)
	for _, node := range osmDocument.Nodes {
		nodesMap[node.Id] = &node
	}

	// Contains a list of pairs of relations
	sitMembers := getSITRelations(&osmDocument)

	// transform sitMembers into CombiData
	combis := make([]CombiData, 0)
	for _, memberPair := range sitMembers {
		combis = append(combis, parseCombiData(memberPair))
	}

	// Create a map from string to CombiLine
	combiLineMap := make(map[string]*CombiLine)
	// Create a map for CombiRoute, grouped by CombiLine.Id
	combiRoutesMap := make(map[int]*CombiRouteContainer)

	// Populate the map
	for _, combi := range combis {
		combiLine := combiLineMap[combi.Company]
		if combiLine == nil {
			// create the company in the map
			combiLine := CombiLine{
				Id:       combi.Id,
				Name:     combi.Company,
				District: "",
				Color:    combi.Color,
			}
			combiLineMap[combi.Company] = &combiLine

			// create the route container
			combiRoutesMap[combi.Id] = &CombiRouteContainer{
				Routes: make([]CombiRoute, 0),
			}
		}
	}

	// Convert each CombiData into a CombiRoute and store it
	for _, combi := range combis {
		log.Printf("Processing %s %s", combi.Company, combi.Ref)
		combiRoute := CombiRoute{
			Name:      combi.Ref,
			Departure: getCoordinates(&osmDocument.Ways, nodesMap, combi.Members),
			Return:    getCoordinates(&osmDocument.Ways, nodesMap, combi.ReturnMembers),
		}
		combiLineSlice := combiRoutesMap[combi.Id]
		combiLineSlice.Routes = append(combiLineSlice.Routes, combiRoute)
	}

	writeOutput(combiLineMap, combiRoutesMap)
}

func getCoordinates(ways *[]OsmWay, nodes map[int64]*OsmNode, membersSlice *[]OsmMember) *[][]float64 {
	coordinates := make([][]float64, 0)

	// filter members, retain only those with type="way"
	newMembers := make([]*OsmMember, 0)
	for _, m := range *membersSlice {
		if m.Type == "way" {
			newMembers = append(newMembers, &m)
		}
	}

	// The coordinate list may be reversed
	// we shold check that the end of the previous set
	// matches the beginning of the next set

	// stores all the osmways in order,
	// without duplicates
	orderedCoordinates := make([]*OsmNode, 0)
	var previousNodeId int64 = -1
	var previousWayId int64 = -1

	// get all coordinates from each
	for _, member := range newMembers {
		// get the way
		way := findOsmway(member.Ref, ways)

		// get all its coordinates
		coords := make([]*OsmNode, 0)
		for _, node := range way.Nds {
			coordinate := findNode(node.Ref, nodes)
			coords = append(coords, coordinate)
		}
		coordsLen := len(coords)

		if previousNodeId == -1 {
			// just add all
			secondWay := findOsmway(newMembers[1].Ref, ways)
			coords = findFirstCoord(way, secondWay, nodes)
			for _, c := range coords {
				orderedCoordinates = append(orderedCoordinates, c)
			}

			// keep track of the last id
			previousNodeId = coords[coordsLen-1].Id
			previousWayId = way.Id
		} else if coords[0].Id == previousNodeId {
			// check if the first coordinate is the same as the previous stored
			// if so, add them except the first one,
			// and keep track of the last one
			for i := 1; i < coordsLen; i += 1 {
				orderedCoordinates = append(orderedCoordinates, coords[i])
			}

			// keep track of the last id
			previousNodeId = coords[coordsLen-1].Id
			previousWayId = way.Id
		} else if coords[coordsLen-1].Id == previousNodeId {
			// if not, they are reversed
			// add all except the last
			for i := coordsLen - 2; i >= 0; i -= 1 {
				orderedCoordinates = append(orderedCoordinates, coords[i])
			}

			// keep track of the last id
			previousNodeId = coords[0].Id
			previousWayId = way.Id
		} else {
			log.Fatalf("Found a way that didn't connect to the previous way. Expected to find node(%d) in way(%d), the previous way id is (%d)", previousNodeId, way.Id, previousWayId)
		}
	}

	// Now compile all the coordinates
	for _, coordinate := range orderedCoordinates {
		coordsSlice := make([]float64, 2)
		coordsSlice[0] = coordinate.Lat
		coordsSlice[1] = coordinate.Lon
		coordinates = append(coordinates, coordsSlice)
	}

	return &coordinates
}

func findFirstCoord(firstWay, secondWay *OsmWay, nodes map[int64]*OsmNode) []*OsmNode {
	// find the common point
	first := make([]*OsmNode, 0)
	for _, node := range firstWay.Nds {
		coordinate := findNode(node.Ref, nodes)
		first = append(first, coordinate)
	}
	firstLast := len(first) - 1
	second := make([]*OsmNode, 0)
	for _, node := range secondWay.Nds {
		coordinate := findNode(node.Ref, nodes)
		second = append(second, coordinate)
	}
	secondLast := len(second) - 1

	if first[firstLast].Id == second[0].Id || first[firstLast].Id == second[secondLast].Id {
		return first
	} else if first[0].Id == second[0].Id || first[0].Id == second[secondLast].Id {
		// reverse the slice
		newFirst := make([]*OsmNode, len(first))
		for i := 0; i < len(first); i += 1 {
			newFirst[i] = first[firstLast-i]
		}

		return newFirst
	} else {
		log.Fatalf("Could not find 2 points in common between 2 ways with ids %d & %d", firstWay.Id, secondWay.Id)
		panic("")
	}
}

func findNode(id int64, nodes map[int64]*OsmNode) *OsmNode {
	node := nodes[id]
	if node == nil {
		log.Fatalf("Node with id %s not found", id)
	}
	return node
}

func findOsmway(id int64, ways *[]OsmWay) *OsmWay {
	var way *OsmWay
	for _, currentWay := range *ways {
		if currentWay.Id == id {
			way = &currentWay
		}
	}
	if way == nil {
		log.Fatalf("Way with id %d not found", id)
	}
	return way
}

func writeOutput(lines map[string]*CombiLine, routes map[int]*CombiRouteContainer) {
	// Create output folder
	os.MkdirAll("output", os.ModePerm)

	//
	// Write the lines JSON file
	//

	// convert the map into an array
	combiLineSlice := make([]*CombiLine, 0)
	for _, combiLine := range lines {
		combiLineSlice = append(combiLineSlice, combiLine)
	}
	// sort the map
	sort.Slice(combiLineSlice, func(i, j int) bool {
		return combiLineSlice[i].Id < combiLineSlice[j].Id
	})

	// print JSON
	jsonBytes, err := json.Marshal(combiLineSlice)
	if err != nil {
		panic(err)
	}
	os.WriteFile("output/lines.json", jsonBytes, 0644)

	//
	// Write each JSON file for the routes
	//
	for lineId, routeContainer := range routes {
		outFilename := fmt.Sprintf("output/routes_%d.json", lineId)
		sort.Slice(routeContainer.Routes, func(i, j int) bool {
			return routeContainer.Routes[i].Name < routeContainer.Routes[j].Name
		})
		jsonBytes, err := json.Marshal(routeContainer.Routes)
		if err != nil {
			panic(nil)
		}
		// write
		err = os.WriteFile(outFilename, jsonBytes, 0644)
		if err != nil {
			panic(err)
		}
	}

	log.Print("JSON files written to output/")
}

func parseCombiData(combiPair *CombiPair) CombiData {
	member := combiPair.Departure
	returnMember := combiPair.Return

	var operatorTag *OsmTag
	var nameTag *OsmTag
	var refTag *OsmTag
	var colorTag *OsmTag

	for _, tag := range member.Tags {
		if tag.K == "operator" {
			operatorTag = &tag
			continue
		}
		if tag.K == "name" {
			nameTag = &tag
			continue
		}
		if tag.K == "ref" {
			refTag = &tag
			continue
		}
		if tag.K == "colour" {
			colorTag = &tag
			continue
		}
	}

	if operatorTag == nil {
		log.Fatalf("Found a SIT member without an operator tag, with id %d\n", member.Id)
	}
	if nameTag == nil {
		log.Fatalf("Found a SIT member without a name tag, with id %d\n", member.Id)
	}
	if refTag == nil {
		log.Fatalf("Found a SIT member without a ref tag, with id %d\n", member.Id)
	}
	if colorTag == nil {
		log.Fatalf("Found a SIT member without a colour tag, with id %d\n", member.Id)
	}

	return CombiData{
		Id:            parseLineId(operatorTag.V),
		Company:       operatorTag.V,
		Name:          nameTag.V,
		Ref:           refTag.V,
		Color:         colorTag.V,
		Members:       &member.Members,
		ReturnMembers: &returnMember.Members,
	}
}

// Finds all the relations that are related to the SIT
func getSITRelations(osmDocument *OsmDocument) []*CombiPair {
	// a route is identified by the pair operator,ref

	// how to know which route is the departure,
	// and which is the return?
	// each district (route, color) is assigned to a district/area
	// in arequipa,
	// so to know which way is each route, we compare the first
	// coordinate against a coordinate located at that district
	// TODO: implement that

	// map of maps
	// map of `operator` to (map of `ref` to OsmRelation)
	operators := make(map[string]*(map[string]*[]*OsmRelation))

	// search, filter and store
	for _, r := range osmDocument.Relations {
		if r.Tag("type", "route") && r.Tag("route", "bus") && r.Tag("network", "SIT") {
			operator := r.GetTag("operator")
			ref := r.GetTag("ref")

			// check if operator map exists, create if not
			operatorMap := operators[operator]
			if operatorMap == nil {
				newMap := make(map[string]*[]*OsmRelation)
				operatorMap = &newMap
				operators[operator] = &newMap
			}

			// check if ref exists, create if not
			refs := (*operatorMap)[ref]
			if refs == nil {
				refsArr := make([]*OsmRelation, 0)
				refs = &refsArr
				(*operatorMap)[ref] = &refsArr
			}

			// insert
			*refs = append(*refs, &r)
		}
	}

	// map and filter
	pairs := make([]*CombiPair, 0)
	for operatorKey, operator := range operators {
		for refKey, relationSlice := range *operator {
			if len(*relationSlice) != 2 {
				log.Printf("operator(%s) ref(%s): expected 2 ref elements, found %d", operatorKey, refKey, len(*relationSlice))
				continue
			}

			// AYAYA!
			newPair := CombiPair{
				Departure: (*relationSlice)[0],
				Return:    (*relationSlice)[1],
			}
			pairs = append(pairs, &newPair)
		}
	}

	return pairs
}

func (r *OsmRelation) HasTag(tagName string) bool {
	for _, tag := range r.Tags {
		if tag.K == tagName {
			return true
		}
	}
	return false
}

func (r *OsmRelation) Tag(tagName string, tagValue string) bool {
	for _, tag := range r.Tags {
		if tag.K == tagName && tag.V == tagValue {
			return true
		}
	}
	return false
}

func (r *OsmRelation) GetTag(tagName string) string {
	for _, tag := range r.Tags {
		if tag.K == tagName {
			return tag.V
		}
	}
	panic(fmt.Sprintf("Tried to get, from relation(%d), tag %s, but it was not found", r.Id, tagName))
}

// Extracts the id from a  line name.
// E.g.: "C11 - Cotum" -> 11
func parseLineId(lineName string) int {
	regex := regexp.MustCompile("C(\\d+) - .+")
	groups := regex.FindStringSubmatch(lineName)
	if groups == nil {
		panic(fmt.Sprintf("Found an invalid line name (doesnt match format): %s", lineName))
	}

	number, err := strconv.Atoi(groups[1])
	if err != nil {
		panic(err)
	}

	return number
}