Fixes to enable following pixelfed

add requested list in actor (holds the follow requests that
haven't been rejected or accepted yet), load actor from memory
instead of disk when there's a new activity in our inbox and
other minor fixes

TODO

@@ -14,7 +14,7 @@
         [✔] When followed, the handler should write the new follower to file
         [✔] Make sure we send our boosts to all our followers
 [ ] Write incoming activities to disk (do we have to?)        
-[ ] Write all the announcements (boosts) to the database to 
+[✔] Write all the announcements (boosts) to the database to 
     their correct actors
 [✔] Check if we are already following users
 [✔] On GetOutbox read the database and present a list of the
@@ -25,7 +25,7 @@
 [ ] Check if we're boosting only stuff from actors we follow, not whatever comes
     through in our inbox
 [✔] Boost not only articles but other things too
-[ ] Sanitize input, never allow slashes or dots
+[✔] Sanitize input, never allow slashes or dots
 [✔] Add summary to actors.json
 [ ] Check local actor names for characters illegal for filenames and ban them
 [✔] Create debug flag

actor.go

@@ -30,10 +30,11 @@ import (
 // Actor represents a local actor we can act on
 // behalf of.
 type Actor struct {
-	name, summary, actorType, iri  string
+	Name, summary, actorType, iri  string
 	followersIRI                   string
 	nuIri                          *url.URL
 	followers, following, rejected map[string]interface{}
+	requested                      map[string]interface{}
 	posts                          map[int]map[string]interface{}
 	publicKey                      crypto.PublicKey
 	privateKey                     crypto.PrivateKey
@@ -41,6 +42,7 @@ type Actor struct {
 	privateKeyPem                  string
 	publicKeyID                    string
 	OnFollow                       func(map[string]interface{})
+	OnReceiveContent               func(map[string]interface{})
 }
 
 // ActorToSave is a stripped down actor representation
@@ -49,7 +51,7 @@ type Actor struct {
 // see https://stackoverflow.com/questions/26327391/json-marshalstruct-returns
 type ActorToSave struct {
 	Name, Summary, ActorType, IRI, PublicKey, PrivateKey string
-	Followers, Following, Rejected                       map[string]interface{}
+	Followers, Following, Rejected, Requested            map[string]interface{}
 }
 
 // MakeActor returns a new local actor we can act
@@ -58,6 +60,7 @@ func MakeActor(name, summary, actorType string) (Actor, error) {
 	followers := make(map[string]interface{})
 	following := make(map[string]interface{})
 	rejected := make(map[string]interface{})
+	requested := make(map[string]interface{})
 	followersIRI := baseURL + name + "/followers"
 	publicKeyID := baseURL + name + "#main-key"
 	iri := baseURL + name
@@ -67,7 +70,7 @@ func MakeActor(name, summary, actorType string) (Actor, error) {
 		return Actor{}, err
 	}
 	actor := Actor{
-		name:         name,
+		Name:         name,
 		summary:      summary,
 		actorType:    actorType,
 		iri:          iri,
@@ -75,12 +78,14 @@ func MakeActor(name, summary, actorType string) (Actor, error) {
 		followers:    followers,
 		following:    following,
 		rejected:     rejected,
+		requested:    requested,
 		followersIRI: followersIRI,
 		publicKeyID:  publicKeyID,
 	}
 
 	// set auto accept by default (this could be a configuration value)
 	actor.OnFollow = func(activity map[string]interface{}) { actor.Accept(activity) }
+	actor.OnReceiveContent = func(activity map[string]interface{}) {}
 
 	// create actor's keypair
 	rng := rand.Reader
@@ -138,6 +143,7 @@ func LoadActor(name string) (Actor, error) {
 	jsonFile := storage + slash + "actors" + slash + name + slash + name + ".json"
 	fileHandle, err := os.Open(jsonFile)
 	if os.IsNotExist(err) {
+		log.Info(name)
 		log.Info("We don't have this kind of actor stored")
 		return Actor{}, err
 	}
@@ -182,7 +188,7 @@ func LoadActor(name string) (Actor, error) {
 	}
 
 	actor := Actor{
-		name:          name,
+		Name:          name,
 		summary:       jsonData["Summary"].(string),
 		actorType:     jsonData["ActorType"].(string),
 		iri:           jsonData["IRI"].(string),
@@ -190,6 +196,7 @@ func LoadActor(name string) (Actor, error) {
 		followers:     jsonData["Followers"].(map[string]interface{}),
 		following:     jsonData["Following"].(map[string]interface{}),
 		rejected:      jsonData["Rejected"].(map[string]interface{}),
+		requested:     jsonData["Requested"].(map[string]interface{}),
 		publicKey:     publicKey,
 		privateKey:    privateKey,
 		publicKeyPem:  jsonData["PublicKey"].(string),
@@ -199,6 +206,7 @@ func LoadActor(name string) (Actor, error) {
 	}
 
 	actor.OnFollow = func(activity map[string]interface{}) { actor.Accept(activity) }
+	actor.OnReceiveContent = func(activity map[string]interface{}) {}
 
 	return actor, nil
 }
@@ -245,19 +253,20 @@ func (a *Actor) save() error {
 
 	// check if we already have a directory to save actors
 	// and if not, create it
-	dir := storage + slash + "actors" + slash + a.name + slash + "items"
+	dir := storage + slash + "actors" + slash + a.Name + slash + "items"
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
 		os.MkdirAll(dir, 0755)
 	}
 
 	actorToSave := ActorToSave{
-		Name:       a.name,
+		Name:       a.Name,
 		Summary:    a.summary,
 		ActorType:  a.actorType,
 		IRI:        a.iri,
 		Followers:  a.followers,
 		Following:  a.following,
 		Rejected:   a.rejected,
+		Requested:  a.requested,
 		PublicKey:  a.publicKeyPem,
 		PrivateKey: a.privateKeyPem,
 	}
@@ -269,7 +278,7 @@ func (a *Actor) save() error {
 	}
 	// log.Info(actorToSave)
 	// log.Info(string(actorJSON))
-	err = ioutil.WriteFile(storage+slash+"actors"+slash+a.name+slash+a.name+".json", actorJSON, 0644)
+	err = ioutil.WriteFile(storage+slash+"actors"+slash+a.Name+slash+a.Name+".json", actorJSON, 0644)
 	if err != nil {
 		log.Printf("WriteFileJson ERROR: %+v", err)
 		return err
@@ -279,19 +288,19 @@ func (a *Actor) save() error {
 }
 
 func (a *Actor) whoAmI() string {
-	return `{"@context":	"https://www.w3.org/ns/activitystreams",
+	return `{"@context":["https://www.w3.org/ns/activitystreams"],
 	"type": "` + a.actorType + `",
-	"id": "` + baseURL + a.name + `",
+	"id": "` + baseURL + a.Name + `",
-	"name": "` + a.name + `",
+	"name": "` + a.Name + `",
-	"preferredUsername": "` + a.name + `",
+	"preferredUsername": "` + a.Name + `",
 	"summary": "` + a.summary + `",
-	"inbox": "` + baseURL + a.name + `/inbox",
+	"inbox": "` + baseURL + a.Name + `/inbox",
-	"outbox": "` + baseURL + a.name + `/outbox",
+	"outbox": "` + baseURL + a.Name + `/outbox",
-	"followers": "` + baseURL + a.name + `/peers/followers",
+	"followers": "` + baseURL + a.Name + `/peers/followers",
-	"following": "` + baseURL + a.name + `/peers/following",
+	"following": "` + baseURL + a.Name + `/peers/following",
 	"publicKey": {
-		"id": "` + baseURL + a.name + `#main-key",
+		"id": "` + baseURL + a.Name + `#main-key",
-		"owner": "` + baseURL + a.name + `",
+		"owner": "` + baseURL + a.Name + `",
 		"publicKeyPem": "` + strings.ReplaceAll(a.publicKeyPem, "\n", "\\n") + `"
 	  }
 	}`
@@ -299,12 +308,12 @@ func (a *Actor) whoAmI() string {
 
 func (a *Actor) newItemID() (hash string, url string) {
 	hash = uniuri.New()
-	return hash, baseURL + a.name + "/item/" + hash
+	return hash, baseURL + a.Name + "/item/" + hash
 }
 
 func (a *Actor) newID() (hash string, url string) {
 	hash = uniuri.New()
-	return hash, baseURL + a.name + "/" + hash
+	return hash, baseURL + a.Name + "/" + hash
 }
 
 // TODO Reply(content string, inReplyTo string)
@@ -324,11 +333,11 @@ func (a *Actor) CreateNote(content, inReplyTo string) {
 	create := make(map[string]interface{})
 	note := make(map[string]interface{})
 	create["@context"] = context()
-	create["actor"] = baseURL + a.name
+	create["actor"] = baseURL + a.Name
 	create["cc"] = a.followersIRI
 	create["id"] = id
 	create["object"] = note
-	note["attributedTo"] = baseURL + a.name
+	note["attributedTo"] = baseURL + a.Name
 	note["cc"] = a.followersIRI
 	note["content"] = content
 	if inReplyTo != "" {
@@ -357,7 +366,7 @@ func (a *Actor) CreateNote(content, inReplyTo string) {
 func (a *Actor) saveItem(hash string, content map[string]interface{}) error {
 	JSON, _ := json.MarshalIndent(content, "", "\t")
 
-	dir := storage + slash + "actors" + slash + a.name + slash + "items"
+	dir := storage + slash + "actors" + slash + a.Name + slash + "items"
 	err := ioutil.WriteFile(dir+slash+hash+".json", JSON, 0644)
 	if err != nil {
 		log.Printf("WriteFileJson ERROR: %+v", err)
@@ -367,7 +376,7 @@ func (a *Actor) saveItem(hash string, content map[string]interface{}) error {
 }
 
 func (a *Actor) loadItem(hash string) (item map[string]interface{}, err error) {
-	dir := storage + slash + "actors" + slash + a.name + slash + "items"
+	dir := storage + slash + "actors" + slash + a.Name + slash + "items"
 	jsonFile := dir + slash + hash + ".json"
 	fileHandle, err := os.Open(jsonFile)
 	if os.IsNotExist(err) {
@@ -405,20 +414,20 @@ func (a *Actor) getPeers(page int, who string) (response []byte, err error) {
 		return nil, errors.New("cannot find collection" + who)
 	}
 	themap := make(map[string]interface{})
-	themap["@context"] = "https://www.w3.org/ns/activitystreams"
+	themap["@context"] = context()
 	if page == 0 {
-		themap["first"] = baseURL + a.name + "/" + who + "?page=1"
+		themap["first"] = baseURL + a.Name + "/peers/" + who + "?page=1"
-		themap["id"] = baseURL + a.name + "/" + who
+		themap["id"] = baseURL + a.Name + "/peers/" + who
 		themap["totalItems"] = strconv.Itoa(len(collection))
 		themap["type"] = "OrderedCollection"
 	} else if page == 1 { // implement pagination
-		themap["id"] = baseURL + a.name + who + "?page=" + strconv.Itoa(page)
+		themap["id"] = baseURL + a.Name + who + "?page=" + strconv.Itoa(page)
 		items := make([]string, 0, len(collection))
 		for k := range collection {
 			items = append(items, k)
 		}
 		themap["orderedItems"] = items
-		themap["partOf"] = baseURL + a.name + "/" + who
+		themap["partOf"] = baseURL + a.Name + "/peers/" + who
 		themap["totalItems"] = len(collection)
 		themap["type"] = "OrderedCollectionPage"
 	}
@@ -465,7 +474,8 @@ func (a *Actor) signedHTTPPost(content map[string]interface{}, to string) (err e
 	req.Header.Add("Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
 	req.Header.Add("User-Agent", userAgent+" "+version)
 	req.Header.Add("Host", iri.Host)
-	req.Header.Add("Accept", "application/activity+json")
+	req.Header.Add("Accept", "application/activity+json; charset=utf-8")
+	req.Header.Add("Content-Type", "application/activity+json; charset=utf-8")
 	sum := sha256.Sum256(b)
 	req.Header.Add("Digest",
 		fmt.Sprintf("SHA-256=%s",
@@ -550,7 +560,7 @@ func (a *Actor) appendToOutbox(iri string) (err error) {
 	// create outbox file if it doesn't exist
 	var outbox *os.File
 
-	outboxFilePath := storage + slash + "actors" + slash + a.name + slash + "outbox.txt"
+	outboxFilePath := storage + slash + "actors" + slash + a.Name + slash + "outbox.txt"
 	outbox, err = os.OpenFile(outboxFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
 		log.Info("Cannot create or open outbox file")
@@ -616,7 +626,10 @@ func (a *Actor) Follow(user string) (err error) {
 				}
 				// save the activity
 				a.saveItem(hash, follow)
-				// we are going to save only on accept so look at
+				a.requested[user] = hash
+				a.save()
+				// we are going to save the request here
+				// and save the follow only on accept so look at
 				// the http handler for the accept code
 			}()
 		}
@@ -760,3 +773,21 @@ func (a *Actor) Accept(follow map[string]interface{}) {
 	go a.signedHTTPPost(accept, follower.inbox)
 
 }
+
+// Followers returns the list of followers
+func (a *Actor) Followers() map[string]string {
+	f := make(map[string]string)
+	for follower, inbox := range a.followers {
+		f[follower] = inbox.(string)
+	}
+	return f
+}
+
+// Following returns the list of followers
+func (a *Actor) Following() map[string]string {
+	f := make(map[string]string)
+	for followee, hash := range a.following {
+		f[followee] = hash.(string)
+	}
+	return f
+}

http.go

@@ -14,7 +14,7 @@ import (
 )
 
 // Serve starts an http server with all the required handlers
-func Serve() {
+func Serve(actors map[string]Actor) {
 
 	var webfingerHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("content-type", "application/jrd+json; charset=utf-8")
@@ -35,13 +35,13 @@ func Serve() {
 
 		responseMap := make(map[string]interface{})
 
-		responseMap["subject"] = "acct:" + actor.name + "@" + server
+		responseMap["subject"] = "acct:" + actor.Name + "@" + server
 		// links is a json array with a single element
 		var links [1]map[string]string
 		link1 := make(map[string]string)
 		link1["rel"] = "self"
 		link1["type"] = "application/activity+json"
-		link1["href"] = baseURL + actor.name
+		link1["href"] = baseURL + actor.Name
 		links[0] = link1
 		responseMap["links"] = links
 
@@ -93,7 +93,7 @@ func Serve() {
 		}
 		postsPerPage := 100
 		var response []byte
-		filename := storage + slash + "actors" + slash + actor.name + slash + "outbox.txt"
+		filename := storage + slash + "actors" + slash + actor.Name + slash + "outbox.txt"
 		totalLines, err := lineCounter(filename)
 		if err != nil {
 			log.Info("Can't read outbox.txt")
@@ -104,9 +104,9 @@ func Serve() {
 			//TODO fix total items
 			response = []byte(`{
 				"@context" : "https://www.w3.org/ns/activitystreams",
-				"first" : "` + baseURL + actor.name + `/outbox?page=1",
+				"first" : "` + baseURL + actor.Name + `/outbox?page=1",
-				"id" : "` + baseURL + actor.name + `/outbox",
+				"id" : "` + baseURL + actor.Name + `/outbox",
-				"last" : "` + baseURL + actor.name + `/outbox?page=` + strconv.Itoa(totalLines/postsPerPage+1) + `",
+				"last" : "` + baseURL + actor.Name + `/outbox?page=` + strconv.Itoa(totalLines/postsPerPage+1) + `",
 				"totalItems" : ` + strconv.Itoa(totalLines) + `, 
 				"type" : "OrderedCollection"
 			 }`)
@@ -124,15 +124,15 @@ func Serve() {
 			}
 			responseMap := make(map[string]interface{})
 			responseMap["@context"] = context()
-			responseMap["id"] = baseURL + actor.name + "/outbox?page=" + pageStr
+			responseMap["id"] = baseURL + actor.Name + "/outbox?page=" + pageStr
 
 			if page*postsPerPage < totalLines {
-				responseMap["next"] = baseURL + actor.name + "/outbox?page=" + strconv.Itoa(page+1)
+				responseMap["next"] = baseURL + actor.Name + "/outbox?page=" + strconv.Itoa(page+1)
 			}
 			if page > 1 {
-				responseMap["prev"] = baseURL + actor.name + "/outbox?page=" + strconv.Itoa(page-1)
+				responseMap["prev"] = baseURL + actor.Name + "/outbox?page=" + strconv.Itoa(page-1)
 			}
-			responseMap["partOf"] = baseURL + actor.name + "/outbox"
+			responseMap["partOf"] = baseURL + actor.Name + "/outbox"
 			responseMap["type"] = "OrderedCollectionPage"
 
 			orderedItems := make([]interface{}, 0, postsPerPage)
@@ -144,7 +144,7 @@ func Serve() {
 				// keep the hash
 				hash := parts[len(parts)-1]
 				// build the filename
-				filename := storage + slash + "actors" + slash + actor.name + slash + "items" + slash + hash + ".json"
+				filename := storage + slash + "actors" + slash + actor.Name + slash + "items" + slash + hash + ".json"
 				// open the file
 				activityJSON, err := ioutil.ReadFile(filename)
 				if err != nil {
@@ -208,31 +208,44 @@ func Serve() {
 			id := follow["id"].(string)
 
 			// check if the object of the follow is us
-			if follow["actor"].(string) != baseURL+actor.name {
+			if follow["actor"].(string) != baseURL+actor.Name {
 				log.Info("This is not for us, ignoring")
 				return
 			}
 			// try to get the hash only
-			hash := strings.Replace(id, baseURL+actor.name+"/item/", "", 1)
+			hash := strings.Replace(id, baseURL+actor.Name+"/item/", "", 1)
 			// if there are still slashes in the result this means the
 			// above didn't work
 			if strings.ContainsAny(hash, "/") {
+				// log.Info(follow)
 				log.Info("The id of this follow is probably wrong")
-				return
+				// we could return here but pixelfed returns
+				// the id as http://domain.tld/actor instead of
+				// http://domain.tld/actor/item/hash so this chokes
+				// return
 			}
 
 			// Have we already requested this follow or are we following anybody that
 			// sprays accepts?
-			savedFollowRequest, err := actor.loadItem(hash)
+
-			if err != nil {
+			// pixelfed doesn't return the original follow thus the id is wrong so we
+			// need to just check if we requested this actor
+
+			// pixelfed doesn't return the original follow thus the id is wrong so we
+			// need to just check if we requested this actor
+			if _, ok := actor.requested[acceptor]; !ok {
 				log.Info("We never requested this follow, ignoring the Accept")
 				return
 			}
-			if savedFollowRequest["id"] != id {
+			// if pixelfed fixes https://github.com/pixelfed/pixelfed/issues/1710 we should uncomment
-				log.Info("Id mismatch between Follow request and Accept")
+			// hash is the _ from above
-				return
+
-			}
+			// if hash != id {
+			// 	log.Info("Id mismatch between Follow request and Accept")
+			// 	return
+			// }
 			actor.following[acceptor] = hash
+			delete(actor.requested, acceptor)
 			actor.save()
 		case "Reject":
 			rejector := activity["actor"].(string)
@@ -245,6 +258,13 @@ func Serve() {
 			// we won't try following them again
 			actor.rejected[rejector] = ""
 			actor.save()
+		case "Create":
+			actor, ok := actors[mux.Vars(r)["actor"]] // load the actor from memory
+			if !ok {
+				log.Error("No such actor")
+				return
+			}
+			actor.OnReceiveContent(activity)
 		default:
 
 		}

remoteActor.go

@@ -85,6 +85,7 @@ func get(iri string) (info map[string]interface{}, err error) {
 	if err != nil {
 		log.Info("something went wrong when unmarshalling the json")
 		log.Info(err)
+		return
 	}
 	info = e.(map[string]interface{})
 

setup.go

@@ -21,7 +21,7 @@ const version = "0.99"
 var client = http.Client{}
 
 // Setup sets our environment up
-func Setup(configurationFile string, debug bool) {
+func Setup(configurationFile string, debug bool) *ini.File {
 	// read configuration file (config.ini)
 
 	if configurationFile == "" {
@@ -70,6 +70,8 @@ func Setup(configurationFile string, debug bool) {
 		log.EnableLevel("info")
 		printer.EnableLevel("info")
 	}
+
+	return cfg
 }
 
 // SetupStorage creates storage