From 6a02d08d5d7cb104f0d580a188227eaa7eb4e95a Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Fri, 20 Sep 2019 16:21:21 +0300 Subject: [PATCH] Fixes to enable following pixelfed add requested list in actor (holds the follow requests that haven't been rejected or accepted yet), load actor from memory instead of disk when there's a new activity in our inbox and other minor fixes --- TODO | 4 +-- actor.go | 93 +++++++++++++++++++++++++++++++++----------------- http.go | 62 +++++++++++++++++++++------------ remoteActor.go | 1 + setup.go | 4 ++- 5 files changed, 109 insertions(+), 55 deletions(-) diff --git a/TODO b/TODO index e997831..34fb93e 100644 --- a/TODO +++ b/TODO @@ -14,7 +14,7 @@ [✔] When followed, the handler should write the new follower to file [✔] Make sure we send our boosts to all our followers [ ] Write incoming activities to disk (do we have to?) -[ ] Write all the announcements (boosts) to the database to +[✔] Write all the announcements (boosts) to the database to their correct actors [✔] Check if we are already following users [✔] On GetOutbox read the database and present a list of the @@ -25,7 +25,7 @@ [ ] Check if we're boosting only stuff from actors we follow, not whatever comes through in our inbox [✔] Boost not only articles but other things too -[ ] Sanitize input, never allow slashes or dots +[✔] Sanitize input, never allow slashes or dots [✔] Add summary to actors.json [ ] Check local actor names for characters illegal for filenames and ban them [✔] Create debug flag diff --git a/actor.go b/actor.go index fc5327f..7026215 100644 --- a/actor.go +++ b/actor.go @@ -30,10 +30,11 @@ import ( // Actor represents a local actor we can act on // behalf of. type Actor struct { - name, summary, actorType, iri string + Name, summary, actorType, iri string followersIRI string nuIri *url.URL followers, following, rejected map[string]interface{} + requested map[string]interface{} posts map[int]map[string]interface{} publicKey crypto.PublicKey privateKey crypto.PrivateKey @@ -41,6 +42,7 @@ type Actor struct { privateKeyPem string publicKeyID string OnFollow func(map[string]interface{}) + OnReceiveContent func(map[string]interface{}) } // ActorToSave is a stripped down actor representation @@ -49,7 +51,7 @@ type Actor struct { // see https://stackoverflow.com/questions/26327391/json-marshalstruct-returns type ActorToSave struct { Name, Summary, ActorType, IRI, PublicKey, PrivateKey string - Followers, Following, Rejected map[string]interface{} + Followers, Following, Rejected, Requested map[string]interface{} } // MakeActor returns a new local actor we can act @@ -58,6 +60,7 @@ func MakeActor(name, summary, actorType string) (Actor, error) { followers := make(map[string]interface{}) following := make(map[string]interface{}) rejected := make(map[string]interface{}) + requested := make(map[string]interface{}) followersIRI := baseURL + name + "/followers" publicKeyID := baseURL + name + "#main-key" iri := baseURL + name @@ -67,7 +70,7 @@ func MakeActor(name, summary, actorType string) (Actor, error) { return Actor{}, err } actor := Actor{ - name: name, + Name: name, summary: summary, actorType: actorType, iri: iri, @@ -75,12 +78,14 @@ func MakeActor(name, summary, actorType string) (Actor, error) { followers: followers, following: following, rejected: rejected, + requested: requested, followersIRI: followersIRI, publicKeyID: publicKeyID, } // set auto accept by default (this could be a configuration value) actor.OnFollow = func(activity map[string]interface{}) { actor.Accept(activity) } + actor.OnReceiveContent = func(activity map[string]interface{}) {} // create actor's keypair rng := rand.Reader @@ -138,6 +143,7 @@ func LoadActor(name string) (Actor, error) { jsonFile := storage + slash + "actors" + slash + name + slash + name + ".json" fileHandle, err := os.Open(jsonFile) if os.IsNotExist(err) { + log.Info(name) log.Info("We don't have this kind of actor stored") return Actor{}, err } @@ -182,7 +188,7 @@ func LoadActor(name string) (Actor, error) { } actor := Actor{ - name: name, + Name: name, summary: jsonData["Summary"].(string), actorType: jsonData["ActorType"].(string), iri: jsonData["IRI"].(string), @@ -190,6 +196,7 @@ func LoadActor(name string) (Actor, error) { followers: jsonData["Followers"].(map[string]interface{}), following: jsonData["Following"].(map[string]interface{}), rejected: jsonData["Rejected"].(map[string]interface{}), + requested: jsonData["Requested"].(map[string]interface{}), publicKey: publicKey, privateKey: privateKey, publicKeyPem: jsonData["PublicKey"].(string), @@ -199,6 +206,7 @@ func LoadActor(name string) (Actor, error) { } actor.OnFollow = func(activity map[string]interface{}) { actor.Accept(activity) } + actor.OnReceiveContent = func(activity map[string]interface{}) {} return actor, nil } @@ -245,19 +253,20 @@ func (a *Actor) save() error { // check if we already have a directory to save actors // and if not, create it - dir := storage + slash + "actors" + slash + a.name + slash + "items" + dir := storage + slash + "actors" + slash + a.Name + slash + "items" if _, err := os.Stat(dir); os.IsNotExist(err) { os.MkdirAll(dir, 0755) } actorToSave := ActorToSave{ - Name: a.name, + Name: a.Name, Summary: a.summary, ActorType: a.actorType, IRI: a.iri, Followers: a.followers, Following: a.following, Rejected: a.rejected, + Requested: a.requested, PublicKey: a.publicKeyPem, PrivateKey: a.privateKeyPem, } @@ -269,7 +278,7 @@ func (a *Actor) save() error { } // log.Info(actorToSave) // log.Info(string(actorJSON)) - err = ioutil.WriteFile(storage+slash+"actors"+slash+a.name+slash+a.name+".json", actorJSON, 0644) + err = ioutil.WriteFile(storage+slash+"actors"+slash+a.Name+slash+a.Name+".json", actorJSON, 0644) if err != nil { log.Printf("WriteFileJson ERROR: %+v", err) return err @@ -279,19 +288,19 @@ func (a *Actor) save() error { } func (a *Actor) whoAmI() string { - return `{"@context": "https://www.w3.org/ns/activitystreams", + return `{"@context":["https://www.w3.org/ns/activitystreams"], "type": "` + a.actorType + `", - "id": "` + baseURL + a.name + `", - "name": "` + a.name + `", - "preferredUsername": "` + a.name + `", + "id": "` + baseURL + a.Name + `", + "name": "` + a.Name + `", + "preferredUsername": "` + a.Name + `", "summary": "` + a.summary + `", - "inbox": "` + baseURL + a.name + `/inbox", - "outbox": "` + baseURL + a.name + `/outbox", - "followers": "` + baseURL + a.name + `/peers/followers", - "following": "` + baseURL + a.name + `/peers/following", + "inbox": "` + baseURL + a.Name + `/inbox", + "outbox": "` + baseURL + a.Name + `/outbox", + "followers": "` + baseURL + a.Name + `/peers/followers", + "following": "` + baseURL + a.Name + `/peers/following", "publicKey": { - "id": "` + baseURL + a.name + `#main-key", - "owner": "` + baseURL + a.name + `", + "id": "` + baseURL + a.Name + `#main-key", + "owner": "` + baseURL + a.Name + `", "publicKeyPem": "` + strings.ReplaceAll(a.publicKeyPem, "\n", "\\n") + `" } }` @@ -299,12 +308,12 @@ func (a *Actor) whoAmI() string { func (a *Actor) newItemID() (hash string, url string) { hash = uniuri.New() - return hash, baseURL + a.name + "/item/" + hash + return hash, baseURL + a.Name + "/item/" + hash } func (a *Actor) newID() (hash string, url string) { hash = uniuri.New() - return hash, baseURL + a.name + "/" + hash + return hash, baseURL + a.Name + "/" + hash } // TODO Reply(content string, inReplyTo string) @@ -324,11 +333,11 @@ func (a *Actor) CreateNote(content, inReplyTo string) { create := make(map[string]interface{}) note := make(map[string]interface{}) create["@context"] = context() - create["actor"] = baseURL + a.name + create["actor"] = baseURL + a.Name create["cc"] = a.followersIRI create["id"] = id create["object"] = note - note["attributedTo"] = baseURL + a.name + note["attributedTo"] = baseURL + a.Name note["cc"] = a.followersIRI note["content"] = content if inReplyTo != "" { @@ -357,7 +366,7 @@ func (a *Actor) CreateNote(content, inReplyTo string) { func (a *Actor) saveItem(hash string, content map[string]interface{}) error { JSON, _ := json.MarshalIndent(content, "", "\t") - dir := storage + slash + "actors" + slash + a.name + slash + "items" + dir := storage + slash + "actors" + slash + a.Name + slash + "items" err := ioutil.WriteFile(dir+slash+hash+".json", JSON, 0644) if err != nil { log.Printf("WriteFileJson ERROR: %+v", err) @@ -367,7 +376,7 @@ func (a *Actor) saveItem(hash string, content map[string]interface{}) error { } func (a *Actor) loadItem(hash string) (item map[string]interface{}, err error) { - dir := storage + slash + "actors" + slash + a.name + slash + "items" + dir := storage + slash + "actors" + slash + a.Name + slash + "items" jsonFile := dir + slash + hash + ".json" fileHandle, err := os.Open(jsonFile) if os.IsNotExist(err) { @@ -405,20 +414,20 @@ func (a *Actor) getPeers(page int, who string) (response []byte, err error) { return nil, errors.New("cannot find collection" + who) } themap := make(map[string]interface{}) - themap["@context"] = "https://www.w3.org/ns/activitystreams" + themap["@context"] = context() if page == 0 { - themap["first"] = baseURL + a.name + "/" + who + "?page=1" - themap["id"] = baseURL + a.name + "/" + who + themap["first"] = baseURL + a.Name + "/peers/" + who + "?page=1" + themap["id"] = baseURL + a.Name + "/peers/" + who themap["totalItems"] = strconv.Itoa(len(collection)) themap["type"] = "OrderedCollection" } else if page == 1 { // implement pagination - themap["id"] = baseURL + a.name + who + "?page=" + strconv.Itoa(page) + themap["id"] = baseURL + a.Name + who + "?page=" + strconv.Itoa(page) items := make([]string, 0, len(collection)) for k := range collection { items = append(items, k) } themap["orderedItems"] = items - themap["partOf"] = baseURL + a.name + "/" + who + themap["partOf"] = baseURL + a.Name + "/peers/" + who themap["totalItems"] = len(collection) themap["type"] = "OrderedCollectionPage" } @@ -465,7 +474,8 @@ func (a *Actor) signedHTTPPost(content map[string]interface{}, to string) (err e req.Header.Add("Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT") req.Header.Add("User-Agent", userAgent+" "+version) req.Header.Add("Host", iri.Host) - req.Header.Add("Accept", "application/activity+json") + req.Header.Add("Accept", "application/activity+json; charset=utf-8") + req.Header.Add("Content-Type", "application/activity+json; charset=utf-8") sum := sha256.Sum256(b) req.Header.Add("Digest", fmt.Sprintf("SHA-256=%s", @@ -550,7 +560,7 @@ func (a *Actor) appendToOutbox(iri string) (err error) { // create outbox file if it doesn't exist var outbox *os.File - outboxFilePath := storage + slash + "actors" + slash + a.name + slash + "outbox.txt" + outboxFilePath := storage + slash + "actors" + slash + a.Name + slash + "outbox.txt" outbox, err = os.OpenFile(outboxFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { log.Info("Cannot create or open outbox file") @@ -616,7 +626,10 @@ func (a *Actor) Follow(user string) (err error) { } // save the activity a.saveItem(hash, follow) - // we are going to save only on accept so look at + a.requested[user] = hash + a.save() + // we are going to save the request here + // and save the follow only on accept so look at // the http handler for the accept code }() } @@ -760,3 +773,21 @@ func (a *Actor) Accept(follow map[string]interface{}) { go a.signedHTTPPost(accept, follower.inbox) } + +// Followers returns the list of followers +func (a *Actor) Followers() map[string]string { + f := make(map[string]string) + for follower, inbox := range a.followers { + f[follower] = inbox.(string) + } + return f +} + +// Following returns the list of followers +func (a *Actor) Following() map[string]string { + f := make(map[string]string) + for followee, hash := range a.following { + f[followee] = hash.(string) + } + return f +} diff --git a/http.go b/http.go index 141624c..dca1e8b 100644 --- a/http.go +++ b/http.go @@ -14,7 +14,7 @@ import ( ) // Serve starts an http server with all the required handlers -func Serve() { +func Serve(actors map[string]Actor) { var webfingerHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/jrd+json; charset=utf-8") @@ -35,13 +35,13 @@ func Serve() { responseMap := make(map[string]interface{}) - responseMap["subject"] = "acct:" + actor.name + "@" + server + responseMap["subject"] = "acct:" + actor.Name + "@" + server // links is a json array with a single element var links [1]map[string]string link1 := make(map[string]string) link1["rel"] = "self" link1["type"] = "application/activity+json" - link1["href"] = baseURL + actor.name + link1["href"] = baseURL + actor.Name links[0] = link1 responseMap["links"] = links @@ -93,7 +93,7 @@ func Serve() { } postsPerPage := 100 var response []byte - filename := storage + slash + "actors" + slash + actor.name + slash + "outbox.txt" + filename := storage + slash + "actors" + slash + actor.Name + slash + "outbox.txt" totalLines, err := lineCounter(filename) if err != nil { log.Info("Can't read outbox.txt") @@ -104,9 +104,9 @@ func Serve() { //TODO fix total items response = []byte(`{ "@context" : "https://www.w3.org/ns/activitystreams", - "first" : "` + baseURL + actor.name + `/outbox?page=1", - "id" : "` + baseURL + actor.name + `/outbox", - "last" : "` + baseURL + actor.name + `/outbox?page=` + strconv.Itoa(totalLines/postsPerPage+1) + `", + "first" : "` + baseURL + actor.Name + `/outbox?page=1", + "id" : "` + baseURL + actor.Name + `/outbox", + "last" : "` + baseURL + actor.Name + `/outbox?page=` + strconv.Itoa(totalLines/postsPerPage+1) + `", "totalItems" : ` + strconv.Itoa(totalLines) + `, "type" : "OrderedCollection" }`) @@ -124,15 +124,15 @@ func Serve() { } responseMap := make(map[string]interface{}) responseMap["@context"] = context() - responseMap["id"] = baseURL + actor.name + "/outbox?page=" + pageStr + responseMap["id"] = baseURL + actor.Name + "/outbox?page=" + pageStr if page*postsPerPage < totalLines { - responseMap["next"] = baseURL + actor.name + "/outbox?page=" + strconv.Itoa(page+1) + responseMap["next"] = baseURL + actor.Name + "/outbox?page=" + strconv.Itoa(page+1) } if page > 1 { - responseMap["prev"] = baseURL + actor.name + "/outbox?page=" + strconv.Itoa(page-1) + responseMap["prev"] = baseURL + actor.Name + "/outbox?page=" + strconv.Itoa(page-1) } - responseMap["partOf"] = baseURL + actor.name + "/outbox" + responseMap["partOf"] = baseURL + actor.Name + "/outbox" responseMap["type"] = "OrderedCollectionPage" orderedItems := make([]interface{}, 0, postsPerPage) @@ -144,7 +144,7 @@ func Serve() { // keep the hash hash := parts[len(parts)-1] // build the filename - filename := storage + slash + "actors" + slash + actor.name + slash + "items" + slash + hash + ".json" + filename := storage + slash + "actors" + slash + actor.Name + slash + "items" + slash + hash + ".json" // open the file activityJSON, err := ioutil.ReadFile(filename) if err != nil { @@ -208,31 +208,44 @@ func Serve() { id := follow["id"].(string) // check if the object of the follow is us - if follow["actor"].(string) != baseURL+actor.name { + if follow["actor"].(string) != baseURL+actor.Name { log.Info("This is not for us, ignoring") return } // try to get the hash only - hash := strings.Replace(id, baseURL+actor.name+"/item/", "", 1) + hash := strings.Replace(id, baseURL+actor.Name+"/item/", "", 1) // if there are still slashes in the result this means the // above didn't work if strings.ContainsAny(hash, "/") { + // log.Info(follow) log.Info("The id of this follow is probably wrong") - return + // we could return here but pixelfed returns + // the id as http://domain.tld/actor instead of + // http://domain.tld/actor/item/hash so this chokes + // return } // Have we already requested this follow or are we following anybody that // sprays accepts? - savedFollowRequest, err := actor.loadItem(hash) - if err != nil { + + // pixelfed doesn't return the original follow thus the id is wrong so we + // need to just check if we requested this actor + + // pixelfed doesn't return the original follow thus the id is wrong so we + // need to just check if we requested this actor + if _, ok := actor.requested[acceptor]; !ok { log.Info("We never requested this follow, ignoring the Accept") return } - if savedFollowRequest["id"] != id { - log.Info("Id mismatch between Follow request and Accept") - return - } + // if pixelfed fixes https://github.com/pixelfed/pixelfed/issues/1710 we should uncomment + // hash is the _ from above + + // if hash != id { + // log.Info("Id mismatch between Follow request and Accept") + // return + // } actor.following[acceptor] = hash + delete(actor.requested, acceptor) actor.save() case "Reject": rejector := activity["actor"].(string) @@ -245,6 +258,13 @@ func Serve() { // we won't try following them again actor.rejected[rejector] = "" actor.save() + case "Create": + actor, ok := actors[mux.Vars(r)["actor"]] // load the actor from memory + if !ok { + log.Error("No such actor") + return + } + actor.OnReceiveContent(activity) default: } diff --git a/remoteActor.go b/remoteActor.go index b5f96a1..7951f43 100644 --- a/remoteActor.go +++ b/remoteActor.go @@ -85,6 +85,7 @@ func get(iri string) (info map[string]interface{}, err error) { if err != nil { log.Info("something went wrong when unmarshalling the json") log.Info(err) + return } info = e.(map[string]interface{}) diff --git a/setup.go b/setup.go index be3dbe2..3830af9 100644 --- a/setup.go +++ b/setup.go @@ -21,7 +21,7 @@ const version = "0.99" var client = http.Client{} // Setup sets our environment up -func Setup(configurationFile string, debug bool) { +func Setup(configurationFile string, debug bool) *ini.File { // read configuration file (config.ini) if configurationFile == "" { @@ -70,6 +70,8 @@ func Setup(configurationFile string, debug bool) { log.EnableLevel("info") printer.EnableLevel("info") } + + return cfg } // SetupStorage creates storage