Adds CORS headers

Let files be delivered by nginx directly

README.md

@@ -1,17 +1,29 @@
 # Prosody Filer
 
-A simple file server for handling XMPP http_upload requests. This server is meat to be used with the Prosody [mod_http_upload_external](https://modules.prosody.im/mod_http_upload_external.html) module.
+A simple file server for handling XMPP http_upload requests. This server is meant to be used with the Prosody [mod_http_upload_external](https://modules.prosody.im/mod_http_upload_external.html) module.
 
-**Why should I use this server?**
+*(This module can also be used with future versions of Ejabberd: https://github.com/processone/ejabberd/commit/fface33d54f24c777dbec96fda6bd00e665327fe)*
 
-* Prosody's integrated http_upload server seems to be memory leaking.
+## Why should I use this server?
+
+* Prosody developers recommend using http_upload_external instead of http_upload (Matthew Wild on the question if http_upload is memory leaking):
+    > "BTW, I am not aware of any memory leaks in the HTTP upload code. However it is known to be very inefficient.
+    > That's why it has a very low upload limit, and **we encourage people to use mod_http_upload_external instead**.
+    > We set out to write a good XMPP server, not  HTTP server (of which many good ones already exist), so our HTTP server is optimised for small bits of data, like BOSH and websocket.
+    > Handling large uploads and downloads was not a goal (and implementing a great HTTP server is not a high priority for the project compared to other things).
+    > **Our HTTP code buffers the entire upload into memory.
+    > More, it does it in an inefficient way that can use up to 4x the actual size of the data (if the data is large).
+    > So uploading a 10MB file can in theory use 40MB RAM.**
+    > But it's not a leak, the RAM is later cleared and reused. [...]
+    > The GC will free the memory at some point, but the OS may still report that Prosody is using that memory due to the way the libc allocator works.
+    > Most long lived processes behave this way (only increasing RAM, rarely decreasing)."
 * This server works without any script interpreters or additional dependencies. It is delivered as a binary.
 * Go is very good at serving HTTP requests.
 
 
-## Download 
+## Download
 
-If you are using regular x86_64 Linux, you can download a finished binary for your system on the [release page](https://github.com/ThomasLeister/prosody-filer/releases). **No need to compile this application yourself**. 
+If you are using regular x86_64 Linux, you can download a finished binary for your system on the [release page](https://github.com/ThomasLeister/prosody-filer/releases). **No need to compile this application yourself**.
 
 
 ## Build (optional)
@@ -20,15 +32,15 @@ If you're using something different than a x64 Linux, you need to compile this a
 
 To compile the server, you need a full Golang development environment. This can be set up quickly: https://golang.org/doc/install#install
 
-Then checkout this repo: 
+Then checkout this repo:
 
     go get github.com/ThomasLeister/prosody-filer
 
-and switch to the new directory: 
+and switch to the new directory:
 
     cd $GOPATH/src/github.com/ThomasLeister/prosody-filer
 
-The application can now be build: 
+The application can now be build:
 
     ### Build static binary
     ./build.sh
@@ -42,7 +54,7 @@ The application can now be build:
 
 ### Setup Prosody Filer environment
 
-Create a new user for Prosody Filer to run as: 
+Create a new user for Prosody Filer to run as:
 
     adduser --disabled-login --disabled-password prosody-filer
 
@@ -50,10 +62,10 @@ Switch to the new user:
 
     su - prosody-filer
 
-Copy  
+Copy
 
-* the binary ```prosody-filer``` and 
-* config ```config.example.toml``` 
+* the binary ```prosody-filer``` and
+* config ```config.example.toml```
 
 to ```/home/prosody-filer/```. Rename the configuration to ```config.toml```.
 
@@ -68,7 +80,7 @@ http_upload_external_secret = "mysecret"
 http_upload_external_file_size_limit = 50000000 -- 50 MB
 ```
 
-Restart Prosody when you are finished: 
+Restart Prosody when you are finished:
 
     systemctl restart prosody
 
@@ -84,7 +96,7 @@ listenport      = "127.0.0.1:5050"
 secret          = "mysecret"
 
 ### Where to store the uploaded files
-storeDir        = "./uploads/"
+storeDir        = "./upload/"
 
 ### Subdirectory for HTTP upload / download requests (usually "upload/")
 uploadSubDir    = "upload/"
@@ -93,6 +105,10 @@ uploadSubDir    = "upload/"
 Make sure ```mysecret``` matches the secret defined in your mod_http_upload_external settings!
 
 
+In addition to that, make sure that the nginx user or group can read the files uploaded
+via prosody-filer if you want to have them served by nginx directly.
+
+
 ### Systemd service file
 
 Create a new Systemd service file: ```/etc/systemd/system/prosody-filer.service```
@@ -107,11 +123,12 @@ Create a new Systemd service file: ```/etc/systemd/system/prosody-filer.service`
     WorkingDirectory=/home/prosody-filer
     User=prosody-filer
     Group=prosody-filer
+    # Group=nginx  # if the files should get served by nginx directly:
 
     [Install]
     WantedBy=multi-user.target
 
-Reload the service definitions, enable the service and start it: 
+Reload the service definitions, enable the service and start it:
 
     systemctl daemon-reload
     systemctl enable prosody-filer
@@ -140,12 +157,13 @@ Create a new config file ```/etc/nginx/sites-available/uploads.myserver.tld```:
 
         location /upload/ {
                 proxy_pass http://127.0.0.1:5050/upload/;
+                proxy_request_buffering off;
         }
     }
 
-Enable the new config:  
+Enable the new config:
 
-    ln -s /etc/ngin/sites-available/uploads.myserver.tld /etc/nginx/sites-enabled/
+    ln -s /etc/nginx/sites-available/uploads.myserver.tld /etc/nginx/sites-enabled/
 
 Check Nginx config:
 
@@ -155,15 +173,44 @@ Reload Nginx:
 
     systemctl reload nginx
 
+#### Configuration for letting nginx serve the uploaded files
 
+```nginx
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+    server_name xmppserver.tld;
+
+    # ...
+
+    location /upload/ {
+        root /home/prosody-filer;
+        client_max_body_size 51m;
+        client_body_buffer_size 51m;
+        try_files $uri $uri/ @prosodyfiler;
+    }
+    location @prosodyfiler {
+        proxy_pass http://127.0.0.1:5050;
+        proxy_buffering off;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+        proxy_set_header X-Forwarded-Server $host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+    }
+
+    # ...
+}
+```
 
 ## Automatic purge
 
 Prosody Filer has no immediate knowlegde over all the stored files and the time they were uploaded, since no database exists for that. Also Prosody is not capable to do auto deletion if *mod_http_upload_external* is used. Therefore the suggested way of purging the uploads directory is to execute a purge command via a cron job:
 
-    @daily    find /var/lib/prosody/uploads -maxdepth 0 -type d -mtime +28 | xargs rm -rf
+    @daily    find /home/prosody-filer/upload/* -maxdepth 0 -type d -mtime +28 | xargs rm -rf
 
-This will delete uploads older than 28 days.  
+This will delete uploads older than 28 days.
 
 
 ## Check if it works
@@ -173,5 +220,3 @@ Get the log via
     journalctl -f -u prosody-filer
 
 If your XMPP clients uploads or downloads any file, there should be some log messages on the screen.
-
-

main.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
@@ -36,6 +37,18 @@ type Config struct {
 
 var conf Config
 
+/*
+ * Sets CORS headers
+ */
+func addCORSheaders(w http.ResponseWriter) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "OPTIONS, HEAD, GET, PUT")
+	w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
+	w.Header().Set("Access-Control-Allow-Credentials", "true")
+	w.Header().Set("Access-Control-Max-Age", "7200")
+}
+
+
 /*
  * Request handler
  * Is activated when a clients requests the file, file information or an upload
@@ -54,7 +67,10 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 		log.Println("Failed to parse URL query params:", err)
 	}
 
-	fileStorePath := strings.TrimLeft(u.Path, conf.UploadSubDir)
+	fileStorePath := strings.TrimPrefix(u.Path, "/" + conf.UploadSubDir)
+
+	// Add CORS headers
+	addCORSheaders(w)
 
 	if r.Method == "PUT" {
 		// Check if MAC is attached to URL
@@ -70,14 +86,11 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 		 * Check if the request is valid
 		 */
 		mac := hmac.New(sha256.New, []byte(conf.Secret))
-		log.Println("Secret:", conf.Secret)
 		log.Println("fileStorePath:", fileStorePath)
 		log.Println("ContentLength:", strconv.FormatInt(r.ContentLength, 10))
 		mac.Write([]byte(fileStorePath + " " + strconv.FormatInt(r.ContentLength, 10)))
 		macString := hex.EncodeToString(mac.Sum(nil))
 
-		log.Println("MAC wanted:", macString)
-
 		/*
 		 * Check whether calculated (expected) MAC is the MAC that client send in "v" URL parameter
 		 */
@@ -85,7 +98,7 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 			// Make sure the path exists
 			os.MkdirAll(filepath.Dir(conf.Storedir+fileStorePath), os.ModePerm)
 
-			file, err := os.Create(conf.Storedir + fileStorePath)
+			file, err := os.OpenFile(conf.Storedir+fileStorePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
 			defer file.Close()
 			if err != nil {
 				log.Println("Creating new file failed:", err)
@@ -96,7 +109,7 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 			n, err := io.Copy(file, r.Body)
 			if err != nil {
 				log.Println("Writing to new file failed:", err)
-				http.Error(w, "409 Conflict", 409)
+				http.Error(w, "500 Internal Server Error", 500)
 				return
 			}
 
@@ -113,9 +126,26 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, "404 Not Found", 404)
 			return
 		}
+
+		/*
+		 * Find out the content type to sent correct header. There is a Go function for retrieving the
+		 * MIME content type, but this does not work with encrypted files (=> OMEMO). Therefore we're just
+		 * relying on file extensions.
+		 */
+		contentType := mime.TypeByExtension(filepath.Ext(fileStorePath))
 		w.Header().Set("Content-Length", strconv.FormatInt(fileinfo.Size(), 10))
+		w.Header().Set("Content-Type", contentType)
 	} else if r.Method == "GET" {
+		contentType := mime.TypeByExtension(filepath.Ext(fileStorePath))
+		if fileStorePath == "" {
+			http.Error(w, "403 Forbidden", 403)
+			return
+		}
+		if contentType == "" {
+			contentType = "application/octet-stream"
+		}
 		http.ServeFile(w, r, conf.Storedir+fileStorePath)
+		w.Header().Set("Content-Type", contentType)
 	} else {
 		log.Println("Invalid method", r.Method, "for access to ", conf.UploadSubDir)
 		http.Error(w, "405 Method Not Allowed", 405)

main_test.go

@@ -1,13 +1,55 @@
 package main
 
+/*
+ * Manual testing with CURL
+ * Send with:
+ * curl -X PUT "http://localhost:5050/upload/thomas/abc/catmetal.jpg?v=e17531b1e88bc9a5cbf816eca8a82fc09969c9245250f3e1b2e473bb564e4be0" --data-binary '@catmetal.jpg'
+ * HMAC: e17531b1e88bc9a5cbf816eca8a82fc09969c9245250f3e1b2e473bb564e4be0
+ */
+
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 )
 
+func mockUpload() {
+	os.MkdirAll(filepath.Dir(conf.Storedir+"thomas/abc/"), os.ModePerm)
+	from, err := os.Open("./catmetal.jpg")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer from.Close()
+
+	to, err := os.OpenFile(conf.Storedir+"thomas/abc/catmetal.jpg", os.O_RDWR|os.O_CREATE, 0660)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer to.Close()
+
+	_, err = io.Copy(to, from)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func cleanup() {
+	// Clean up
+	if _, err := os.Stat(conf.Storedir); err == nil {
+		// Delete existing catmetal picture
+		err := os.RemoveAll(conf.Storedir)
+		if err != nil {
+			log.Println("Error while cleaning up:", err)
+		}
+	}
+}
+
 func TestReadConfig(t *testing.T) {
 	// Set config
 	err := readConfig("config.toml", &conf)
@@ -46,6 +88,9 @@ func TestUploadValid(t *testing.T) {
 	if status := rr.Code; status != http.StatusOK {
 		t.Errorf("handler returned wrong status code: got %v want %v. HTTP body: %s", status, http.StatusOK, rr.Body.String())
 	}
+
+	// clean up
+	cleanup()
 }
 
 func TestUploadMissingMAC(t *testing.T) {
@@ -142,6 +187,9 @@ func TestDownloadHead(t *testing.T) {
 	// Set config
 	readConfig("config.toml", &conf)
 
+	// Mock upload
+	mockUpload()
+
 	// Create request
 	req, err := http.NewRequest("HEAD", "/upload/thomas/abc/catmetal.jpg", nil)
 
@@ -159,12 +207,18 @@ func TestDownloadHead(t *testing.T) {
 	if status := rr.Code; status != http.StatusOK {
 		t.Errorf("handler returned wrong status code: got %v want %v. HTTP body: %s", status, http.StatusOK, rr.Body.String())
 	}
+
+	// cleanup
+	cleanup()
 }
 
 func TestDownloadGet(t *testing.T) {
 	// Set config
 	readConfig("config.toml", &conf)
 
+	// moch upload
+	mockUpload()
+
 	// Create request
 	req, err := http.NewRequest("GET", "/upload/thomas/abc/catmetal.jpg", nil)
 
@@ -182,4 +236,30 @@ func TestDownloadGet(t *testing.T) {
 	if status := rr.Code; status != http.StatusOK {
 		t.Errorf("handler returned wrong status code: got %v want %v. HTTP body: %s", status, http.StatusOK, rr.Body.String())
 	}
+
+	// cleanup
+	cleanup()
+}
+
+func TestEmptyGet(t *testing.T) {
+	// Set config
+	readConfig("config.toml", &conf)
+
+	// Create request
+	req, err := http.NewRequest("GET", "", nil)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	handler := http.HandlerFunc(handleRequest)
+
+	// Send request and record response
+	handler.ServeHTTP(rr, req)
+
+	// Check status code
+	if status := rr.Code; status != http.StatusForbidden {
+		t.Errorf("handler returned wrong status code: got %v want %v. HTTP body: %s", status, http.StatusForbidden, rr.Body.String())
+	}
 }