ZipStream

This commit is contained in:
Roman Hergenreder 2021-01-13 01:36:04 +01:00
parent 8da94bafdf
commit e19b4d1a46
15 changed files with 614 additions and 43 deletions

@ -4,8 +4,11 @@ namespace Api {
use Driver\SQL\Condition\Compare;
use Driver\SQL\Condition\CondIn;
use Driver\SQL\Query\Query;
use Driver\Sql\Condition\CondNull;
use Driver\SQL\SQL;
use External\ZipStream\BufferWriter;
use External\ZipStream\File;
use External\ZipStream\ZipStream;
abstract class FileAPI extends Request {
@ -65,6 +68,27 @@ namespace Api {
}
}
protected function downloadZip($files) {
if ($files == null || empty($files) || !is_array($files)) {
return $this->createError("No files to download");
}
header('Content-Disposition: attachment; filename="files.zip"');
header('Content-Type: application/zip');
$writer = new BufferWriter();
$writer->registerCallback(function ($w) { echo $w->read(); });
$zipStream = new ZipStream($writer);
foreach ($files as $file) {
$f = new File($file["name"]);
$f->loadFromFile($file["path"]);
$zipStream->saveFile($f);
}
$zipStream->close();
exit;
}
protected function &findDirectory(&$files, $id) {
if ($id !== null) {
@ -123,7 +147,7 @@ namespace Api {
$query->innerJoin("UserFileTokenFile", "UserFile.uid", "file_id")
->innerJoin("UserFileToken", "UserFileToken.uid", "token_id")
->where(new Compare("token", $token))
->where(new Compare("valid_until", $sql->now(), ">="));
->where(new CondNull("valid_until"), new Compare("valid_until", $sql->now(), ">="));
}
}
}
@ -137,6 +161,7 @@ namespace Api\File {
use Api\Parameter\StringType;
use Driver\SQL\Condition\Compare;
use Driver\SQL\Condition\CondIn;
use Driver\Sql\Condition\CondNull;
use Objects\User;
class ValidateToken extends FileAPI {
@ -161,7 +186,7 @@ namespace Api\File {
->leftJoin("UserFileTokenFile", "UserFileToken.uid", "token_id")
->leftJoin("UserFile", "UserFile.uid", "file_id")
->where(new Compare("token", $token))
->where(new Compare("valid_until", $sql->now(), ">"))
->where(new CondNull("valid_until"), new Compare("valid_until", $sql->now(), ">="))
->execute();
$this->success = ($res !== false);
@ -420,7 +445,7 @@ namespace Api\File {
$res = $sql->select("uid", "token_type", "maxFiles", "maxSize", "extensions", "user_id")
->from("UserFileToken")
->where(new Compare("token", $token))
->where(new Compare("valid_until", $sql->now(), ">="))
->where(new CondNull("valid_until"), new Compare("valid_until", $sql->now(), ">="))
->limit(1)
->execute();
@ -550,8 +575,8 @@ namespace Api\File {
}
$sql = $this->user->getSQL();
$fileIds = $this->getParam("id");
$query = $sql->select("UserFile.uid", "path", "name")->from("UserFile");
$fileIds = array_unique($this->getParam("id"));
$query = $sql->select("UserFile.uid", "path", "name", "directory")->from("UserFile");
$this->filterFiles($sql, $query, $fileIds, $token);
$res = $query->execute();
@ -559,21 +584,34 @@ namespace Api\File {
$this->lastError = $sql->getLastError();
if ($this->success) {
if (empty($res)) {
$foundFiles = array();
foreach ($res as $row) {
$foundFiles[$row["uid"]] = $row;
}
$filesToDownload = array();
foreach ($fileIds as $fileId) {
if (!array_key_exists($fileId, $foundFiles)) {
if (is_null($token)) {
return $this->createError("File not found");
return $this->createError("File not found: $fileId");
} else {
return $this->createError("Permission denied (token)");
}
} else {
if ($res[0]["directory"]) {
return $this->createError("Cannot download directory (yet)");
} else {
$path = $res[0]["path"];
$name = $res[0]["name"];
$this->downloadFile($name, $path);
} else if (!$foundFiles[$fileId]["directory"]) {
$filesToDownload[] = $foundFiles[$fileId];
}
}
if (empty($filesToDownload)) {
return $this->createError("No file selected");
} else if (count($filesToDownload) === 1) {
$file = array_shift($filesToDownload);
$path = $file["path"];
$name = $file["name"];
$this->downloadFile($name, $path);
} else {
$this->downloadZip($filesToDownload);
}
}
return $this->success;
@ -680,8 +718,7 @@ namespace Api\File {
$sql = $this->user->getSQL();
$token = generateRandomString(36);
$validUntil = (new \DateTime())->modify("+$durability MINUTES");
$validUntil = $durability == 0 ? null : (new \DateTime())->modify("+$durability MINUTES");
$res = $sql->insert("UserFileToken",
array("token", "token_type", "maxSize", "maxFiles", "extensions", "valid_until", "user_id"))
->addRow($token, "upload", $maxSize, $maxFiles, $extensions, $validUntil, $this->user->getId())
@ -758,7 +795,7 @@ namespace Api\File {
// Insert
$token = generateRandomString(36);
$validUntil = (new \DateTime())->modify("+$durability MINUTES");
$validUntil = $durability == 0 ? null : (new \DateTime())->modify("+$durability MINUTES");
$res = $sql->insert("UserFileToken", array("token_type", "valid_until", "user_id", "token"))
->addRow("download", $validUntil, $this->user->getId(), $token)
->returning("uid")

@ -50,7 +50,7 @@ class Request {
foreach($this->params as $name => $param) {
$value = $values[$name] ?? NULL;
$isEmpty = (is_string($value) || is_array($value)) && empty($value);
$isEmpty = (is_string($value) && strlen($value) === 0) || (is_array($value) && empty($value));
if(!$param->optional && (is_null($value) || $isEmpty)) {
return $this->createError("Missing parameter: $name");
}

@ -0,0 +1,16 @@
<?php
namespace Driver\Sql\Condition;
class CondNull extends Condition {
private string $column;
public function __construct(string $col) {
$this->column = $col;
}
public function getColumn() { return $this->column; }
}

@ -9,6 +9,7 @@ use Driver\SQL\Condition\CondIn;
use Driver\SQL\Condition\Condition;
use Driver\SQL\Condition\CondKeyword;
use Driver\SQL\Condition\CondNot;
use Driver\Sql\Condition\CondNull;
use Driver\SQL\Condition\CondOr;
use Driver\SQL\Constraint\Constraint;
use \Driver\SQL\Constraint\Unique;
@ -398,6 +399,8 @@ abstract class SQL {
}
return "NOT $expression";
} else if($condition instanceof CondNull) {
return $this->columnName($condition->getColumn()) . " IS NULL";
} else {
$this->lastError = "Unsupported condition type: " . get_class($condition);
return false;

@ -0,0 +1,62 @@
<?php
/**
* Copyright (c) Borago 2019
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
**/
namespace External\ZipStream {
class BufferWriter implements Writer {
private $stream = '';
private $offset = 0;
private $callback = null;
public function __construct() {
}
public function registerCallback($callback) {
$this->callback = $callback;
}
public function write($data) {
$this->offset += strlen($data);
$this->stream .= $data;
if ($this->callback !== null) {
call_user_func($this->callback, $this);
}
return strlen($data);
}
public function read() {
$data = $this->stream;
$this->stream = '';
return $data;
}
public function offset() {
return $this->offset;
}
public function close() {
if ($this->callback !== null) {
call_user_func($this->callback, $this);
}
}
}
}

207
core/External/ZipStream/File.php vendored Normal file

@ -0,0 +1,207 @@
<?php
/**
* Copyright (c) Borago 2019
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
**/
namespace External\ZipStream {
class File {
private $name;
private $content = '';
private $fileHandle = false;
private $lastModificationTimestamp;
private $crc32 = null;
private $fileSize = 0;
private $compressedSize = 0;
private $offset = 0;
private $bitField = 0;
private $useCompression = true;
private $deflateState = null;
//check for duplications //currently not used
private $sha256;
public const BIT_NO_SIZE_IN_HEADER = 0b0000000000001000;
public const BIT_UTF8_NAMES = 0b0000100000000000;
public function __construct($name) {
$this->name = $name;
$this->lastModificationTimestamp = time();
$this->crc32 = hash('crc32b', '', true);
$this->compressedSize = 0;
$this->bitField = 0;
$this->bitField |= self::BIT_NO_SIZE_IN_HEADER;
$this->bitField |= self::BIT_UTF8_NAMES;
$this->deflateState = deflate_init(ZLIB_ENCODING_RAW, ['level' => 9]);
}
public function setContent($content) {
$this->crc32 = hash('crc32b', $content, true);
$this->sha256 = hash('sha256', $content);
$this->content = $content;
$this->fileSize = strlen($content);
$this->fileHandle = false;
}
public function loadFromFile($filename) {
$this->crc32 = hash_file('crc32b', $filename, true);
$this->sha256 = hash_file('sha256', $filename);
$this->fileSize = filesize($filename);
$this->fileHandle = fopen($filename, 'rb');
}
public function name() {
return $this->name;
}
public function sha256() {
return $this->sha256;
}
private function unixTimeToDosTime($timestamp) {
$hour = intval(date('H', $timestamp));
$min = intval(date('i', $timestamp));
$sec = intval(date('s', $timestamp));
return ($hour << 11) |
($min << 5) |
($sec >> 1);
}
private function unixTimeToDosDate($timestamp) {
$year = intval(date('Y', $timestamp));
$month = intval(date('m', $timestamp));
$day = intval(date('d', $timestamp));
return (($year - 1980) << 9) |
($month << 5) |
($day);
}
public function readLocalFileHeader() {
if (!$this->useCompression) {
$this->compressedSize = $this->fileSize;
}
$header = "";
$header .= "\x50\x4b\x03\x04";
$header .= "\x14\x00"; //version 2.0 and MS-DOS compatible
$header .= pack("v", $this->bitField); //general purpose bit flag
if ($this->useCompression) {
$header .= "\x08\x00"; //compression Method - deflate
} else {
$header .= "\x00\x00"; //compression Method - no
}
$header .= pack("v", $this->unixTimeToDosTime($this->lastModificationTimestamp)); //dos time
$header .= pack("v", $this->unixTimeToDosDate($this->lastModificationTimestamp)); //dos date
if ($this->bitField & self::BIT_NO_SIZE_IN_HEADER) {
$header .= pack("V", 0); //crc32
$header .= pack("V", 0); //compressed Size
$header .= pack("V", 0); //uncompressed Size
} else {
$header .= strrev($this->crc32);
$header .= pack("V", $this->compressedSize); //compressed Size
$header .= pack("V", $this->fileSize); //uncompressed Size
}
$header .= pack("v", strlen($this->name)); //filename
$header .= "\x00\x00"; //extra field length
$header .= $this->name;
return $header;
}
public function readDataDescriptor() {
$data = "";
$data .= "\x50\x4b\x07\x08";
$data .= strrev($this->crc32);
$data .= pack("V", $this->compressedSize); //compressed Size
$data .= pack("V", $this->fileSize); //uncompressed Size
return $data;
}
public function readFileDataImp() {
$ret = null;
if ($this->fileHandle !== false) {
$block = fread($this->fileHandle, 65536);
if (!empty($block)) {
$ret = $block;
}
} else {
$ret = $this->content;
$this->content = null;
}
return $ret;
}
public function readFileData() {
$ret = null;
if ($this->useCompression) {
$block = $this->readFileDataImp();
if ($this->deflateState !== null) {
if ($block !== null) {
$ret = deflate_add($this->deflateState, $block, ZLIB_NO_FLUSH);
} else {
$ret = deflate_add($this->deflateState, '', ZLIB_FINISH);
$this->deflateState = null;
}
}
if ($ret !== null) {
$this->compressedSize += strlen($ret);
}
} else {
$ret = $this->readFileDataImp();
}
return $ret;
}
public function setOffset($offset) {
$this->offset = $offset;
}
public function readCentralDirectoryHeader() {
$header = "";
$header .= "\x50\x4b\x01\x02";
$header .= "\x14\x00"; //version 2.0 and MS-DOS compatible
$header .= "\x14\x00"; //version 2.0 and MS-DOS compatible
$header .= pack("v", $this->bitField); //general purpose bit flag
$header .= "\x00\x00"; //compression Method - no
$header .= pack("v", $this->unixTimeToDosTime($this->lastModificationTimestamp)); //dos time
$header .= pack("v", $this->unixTimeToDosDate($this->lastModificationTimestamp)); //dos date
$header .= strrev($this->crc32);
$header .= pack("V", $this->compressedSize); //compressed Size
$header .= pack("V", $this->fileSize); //uncompressed Size
$header .= pack("v", strlen($this->name)); //filename
$header .= "\x00\x00"; //extra field length
$header .= "\x00\x00"; //comment length
$header .= "\x00\x00"; //disk num start
$header .= "\x00\x00"; //int file attr
$header .= "\x00\x00\x00\x00"; //ext file attr
$header .= pack("V", $this->offset); //relative offset
$header .= $this->name;
return $header;
}
public function closeHandle() {
if ($this->fileHandle) {
fclose($this->fileHandle);
}
}
}
}

63
core/External/ZipStream/FileWriter.php vendored Normal file

@ -0,0 +1,63 @@
<?php
/**
* Copyright (c) Borago 2019
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
**/
namespace External\ZipStream {
class FileWriter implements Writer {
private $offset = 0;
private $fileHandle = false;
public function __construct($filename) {
if (!empty($filename)) {
$this->open($filename);
}
}
public function __destruct() {
$this->close();
}
public function open($filename) {
$this->close();
$this->fileHandle = fopen($filename, 'wb');
}
public function write($data) {
$this->offset += strlen($data);
if ($this->fileHandle === false) {
throw new \Exception('No file opened.');
} else {
return fwrite($this->fileHandle, $data);
}
}
public function offset() {
return $this->offset;
}
public function close() {
if ($this->fileHandle !== false) {
fclose($this->fileHandle);
}
$this->fileHandle = false;
}
}
}

29
core/External/ZipStream/Writer.php vendored Normal file

@ -0,0 +1,29 @@
<?php
/**
* Copyright (c) Borago 2019
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
**/
namespace External\ZipStream {
interface Writer {
public function write($data);
public function close();
public function offset();
}
}

72
core/External/ZipStream/ZipStream.php vendored Normal file

@ -0,0 +1,72 @@
<?php
/**
* Copyright (c) Borago 2019
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
**/
namespace External\ZipStream {
class ZipStream {
private $writer = null;
private $files = [];
public function __construct($writer) {
$this->writer = $writer;
}
public function saveFile($file) {
$isSymlink = false; //currently not used
foreach ($this->files as $f) {
if ($f->name() == $file->name()) {
return false;
}
if ($f->sha256() == $file->sha256()) {
$isSymlink = true;
}
}
$file->setOffset($this->writer->offset());
$this->writer->write($file->readLocalFileHeader());
while (($buffer = $file->readFileData()) !== null) {
$this->writer->write($buffer);
}
$this->writer->write($file->readDataDescriptor());
$this->files[] = $file;
$file->closeHandle();
return true;
}
public function close() {
$size = 0;
$offset = $this->writer->offset();
foreach ($this->files as $file) {
$size += $this->writer->write($file->readCentralDirectoryHeader());
}
$data = "";
$data .= "\x50\x4b\x05\x06";
$data .= "\x00\x00"; //number of disks
$data .= "\x00\x00"; //number of the disk with the start of the central directory
$data .= pack("v", count($this->files)); //total number of entries in the central directory on this disk
$data .= pack("v", count($this->files)); //total number of entries in the central directory
$data .= pack("V", $size); //size of the central directory
$data .= pack("V", $offset); //offset of start of central directory with respect to the starting disk number
$data .= "\x0\x0"; //comment length
$this->writer->write($data);
}
}
}

@ -7751,6 +7751,11 @@
"object.assign": "^4.1.0"
}
},
"keymaster": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/keymaster/-/keymaster-1.6.2.tgz",
"integrity": "sha1-4a5U0OqUiPn2C2a2aPAumhlGxus="
},
"killable": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/killable/-/killable-1.0.1.tgz",

@ -285,7 +285,8 @@ export class FileBrowser extends React.Component {
</tbody>
</table>
<div className={"file-control-buttons"}>
<button type={"button"} className={"btn btn-success"} disabled={selectedCount === 0}>
<button type={"button"} className={"btn btn-success"} disabled={selectedCount === 0}
onClick={() => this.onDownload(selectedIds)}>
<Icon icon={"download"} className={"mr-1"}/>
Download Selected Files ({selectedCount})
</button>
@ -305,7 +306,7 @@ export class FileBrowser extends React.Component {
Upload
</button>
<button type={"button"} className={"btn btn-danger"} disabled={selectedCount === 0}
onClick={(e) => this.deleteFiles(selectedIds)}>
onClick={() => this.deleteFiles(selectedIds)}>
<Icon icon={"trash"} className={"mr-1"}/>
Delete Selected Files ({selectedCount})
</button>
@ -354,6 +355,7 @@ export class FileBrowser extends React.Component {
}
deleteFiles(selectedIds) {
if (selectedIds && selectedIds.length > 0) {
let token = (this.state.api.loggedIn ? null : this.state.token.value);
this.state.api.delete(selectedIds, token).then((res) => {
if (res.success) {
@ -363,6 +365,7 @@ export class FileBrowser extends React.Component {
}
});
}
}
onUpload() {
let token = (this.state.api.loggedIn ? null : this.state.token.value);
@ -375,4 +378,31 @@ export class FileBrowser extends React.Component {
}
});
}
onDownload(selectedIds) {
if (selectedIds && selectedIds.length > 0) {
let token = (this.state.api.loggedIn ? "" : "&token=" + this.state.token.value);
let ids = selectedIds.map(id => "id[]=" + id).join("&");
let downloadUrl = "/api/file/download?" + ids + token;
fetch(downloadUrl)
.then(response => {
let header = response.headers.get("Content-Disposition") || "";
let fileNameFields = header.split(";").filter(c => c.trim().toLowerCase().startsWith("filename="));
let fileName = null;
if (fileNameFields.length > 0) {
fileName = fileNameFields[0].trim().substr("filename=".length);
} else {
fileName = null;
}
response.blob().then(blob => {
let url = window.URL.createObjectURL(blob);
let a = document.createElement('a');
a.href = url;
if (fileName !== null) a.download = fileName;
a.click();
});
});
}
}
}

@ -0,0 +1,37 @@
import React from 'react';
class Popup extends React.Component {
constructor(props) {
super(props);
this.state = {
title: props.title || "Title",
content: props.content || "Content",
buttons: props.buttons || ["Ok", "Cancel"]
}
}
render() {
return <div className="modal" tabIndex="-1" role="dialog">
<div className="modal-dialog" role="document">
<div className="modal-content">
<div className="modal-header">
<h5 className="modal-title">{this.state.title}</h5>
<button type="button" className="close" data-dismiss="modal" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
<div className="modal-body">
<p>Modal body text goes here.</p>
</div>
<div className="modal-footer">
<button type="button" className="btn btn-secondary" data-dismiss="modal">Close</button>
<button type="button" className="btn btn-primary">Save changes</button>
</div>
</div>
</div>
</div>;
}
}

@ -1,6 +1,7 @@
import * as React from "react";
import Icon from "./icon";
import moment from "moment";
import Popup from "react-popup";
export class TokenList extends React.Component {
@ -23,12 +24,15 @@ export class TokenList extends React.Component {
});
} else {
for (const token of this.state.tokens) {
const revoked = moment(token.valid_until).isSameOrBefore(new Date());
const validUntil = token.valid_until;
const revoked = validUntil !== null && moment(validUntil).isSameOrBefore(new Date());
const timeStr = (validUntil === null ? "Forever" : moment(validUntil).format("Do MMM YYYY, HH:mm"));
rows.push(
<tr key={"token-" + token.uid} className={revoked ? "token-revoked" : ""}>
<td>{token.token}</td>
<td>{token.type}</td>
<td>{moment(token.valid_until).format("Do MMM YYYY, HH:mm")}</td>
<td>{timeStr}</td>
<td>
<Icon icon={"times"} className={"clickable text-" + (revoked ? "secondary" : "danger")}
onClick={() => (revoked ? null : this.onRevokeToken(token.token) )}
@ -65,7 +69,7 @@ export class TokenList extends React.Component {
</tbody>
</table>
<div>
<button type={"button"} className={"btn btn-success m-2"}>
<button type={"button"} className={"btn btn-success m-2"} onClick={this.onCreateToken.bind(this)}>
<Icon icon={"plus"} className={"mr-1"}/>
Create Token
</button>
@ -94,4 +98,8 @@ export class TokenList extends React.Component {
}
});
}
onCreateToken() {
Popup.alert('I am alert, nice to meet you');
}
}

@ -86,7 +86,8 @@ class FileControlPanel extends React.Component {
</div> :
<></>;
return <div className={"container mt-4"}>
return <>
<div className={"container mt-4"}>
<div className={"row"}>
<div className={"col-lg-8 col-md-10 col-sm-12 mx-auto"}>
<h2>File Control Panel</h2>
@ -94,7 +95,8 @@ class FileControlPanel extends React.Component {
</div>
</div>
{ tokenList }
</div>;
</div>
</>;
} else {
return <div className={"container mt-4"}>
<div className={"row"}>

2
js/files.min.js vendored

File diff suppressed because one or more lines are too long