Browse Source

💥 Move to Postgres from MongoDB

feature/mongodb-to-postgres
poeti8 1 year ago
parent
commit
752dc7cd6c
40 changed files with 3018 additions and 1604 deletions
  1. +4
    -3
      .eslintrc
  2. +3
    -2
      client/components/Stats/Stats.js
  3. +2
    -2
      client/components/Table/TBody/TBody.js
  4. +4
    -4
      client/components/Table/TBody/TBodyCount.js
  5. +9
    -9
      client/components/Table/Table.js
  6. +1
    -13
      client/pages/_document.js
  7. +5
    -3
      client/pages/stats.js
  8. +114
    -0
      global.d.ts
  9. +1
    -1
      nodemon.json
  10. +1040
    -381
      package-lock.json
  11. +28
    -23
      package.json
  12. +0
    -27
      scripts/migration.ts
  13. +21
    -21
      server/configToEnv.ts
  14. +87
    -86
      server/controllers/authController.ts
  15. +160
    -164
      server/controllers/linkController.ts
  16. +80
    -85
      server/controllers/validateBodyController.ts
  17. +2
    -2
      server/cron.ts
  18. +87
    -31
      server/db/domain.ts
  19. +36
    -16
      server/db/host.ts
  20. +37
    -19
      server/db/ip.ts
  21. +314
    -277
      server/db/link.ts
  22. +160
    -102
      server/db/user.ts
  23. +28
    -0
      server/knex.ts
  24. +6
    -6
      server/mail/mail.ts
  25. +67
    -0
      server/migration/01_host.ts
  26. +86
    -0
      server/migration/02_users.ts
  27. +97
    -0
      server/migration/03_domains.ts
  28. +181
    -0
      server/migration/04_links.ts
  29. +27
    -23
      server/models/domain.ts
  30. +21
    -21
      server/models/host.ts
  31. +14
    -16
      server/models/ip.ts
  32. +33
    -30
      server/models/link.ts
  33. +32
    -45
      server/models/user.ts
  34. +75
    -53
      server/models/visit.ts
  35. +0
    -8
      server/module.d.ts
  36. +11
    -11
      server/passport.ts
  37. +5
    -5
      server/redis.ts
  38. +107
    -88
      server/server.ts
  39. +32
    -26
      server/utils/index.ts
  40. +1
    -1
      tsconfig.server.json

+ 4
- 3
.eslintrc View File

@ -1,10 +1,8 @@
{
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/eslint-recommended",
"plugin:@typescript-eslint/recommended",
"prettier",
"prettier/@typescript-eslint"
"plugin:prettier/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
@ -15,6 +13,7 @@
"eqeqeq": ["warn", "always", { "null": "ignore" }],
"no-useless-return": "warn",
"no-var": "warn",
"no-console": "warn",
"max-len": ["warn", { "comments": 80 }],
"no-param-reassign": ["warn", { "props": false }],
"require-atomic-updates": 0,
@ -22,11 +21,13 @@
"@typescript-eslint/no-unused-vars": "off", // "warn" for production
"@typescript-eslint/no-explicit-any": "off", // "warn" for production
"@typescript-eslint/no-var-requires": "off",
"@typescript-eslint/camelcase": "off",
"@typescript-eslint/no-object-literal-type-assertion": "off",
"@typescript-eslint/no-parameter-properties": "off",
"@typescript-eslint/explicit-function-return-type": "off"
},
"env": {
"es6": true,
"browser": true,
"node": true,
"mocha": true

+ 3
- 2
client/components/Stats/Stats.js View File

@ -85,10 +85,10 @@ class Stats extends Component {
}
componentDidMount() {
const { id } = this.props;
const { domain, id } = this.props;
if (!id) return null;
return axios
.get(`/api/url/stats?id=${id}`, { headers: { Authorization: cookie.get('token') } })
.get(`/api/url/stats?id=${id}&domain=${domain}`, { headers: { Authorization: cookie.get('token') } })
.then(({ data }) =>
this.setState({
stats: data,
@ -155,6 +155,7 @@ class Stats extends Component {
Stats.propTypes = {
isAuthenticated: PropTypes.bool.isRequired,
domain: PropTypes.string.isRequired,
id: PropTypes.string.isRequired,
showPageLoading: PropTypes.func.isRequired,
};

+ 2
- 2
client/components/Table/TBody/TBody.js View File

@ -93,13 +93,13 @@ const TableBody = ({ copiedIndex, handleCopy, tableLoading, showModal, urls }) =
<a href={url.target}>{url.target}</a>
</Td>
<Td flex="1" date>
{`${distanceInWordsToNow(url.createdAt)} ago`}
{`${distanceInWordsToNow(url.created_at)} ago`}
</Td>
<Td flex="1" withFade>
<TBodyShortUrl index={index} copiedIndex={copiedIndex} handleCopy={handleCopy} url={url} />
</Td>
<Td flex="1">
<TBodyCount url={url} showModal={showModal} />
<TBodyCount url={url} showModal={showModal(url)} />
</Td>
</tr>
);

+ 4
- 4
client/components/Table/TBody/TBodyCount.js View File

@ -50,9 +50,9 @@ class TBodyCount extends Component {
goTo(e) {
e.preventDefault();
const { id, domain } = this.props.url;
this.props.showLoading();
const host = URL.parse(this.props.url.shortLink).hostname;
Router.push(`/stats?id=${this.props.url.id}${`&domain=${host}`}`);
Router.push(`/stats?id=${id}${domain ? `&domain=${domain}`: ''}`);
}
render() {
@ -61,10 +61,10 @@ class TBodyCount extends Component {
return (
<Wrapper>
{url.count || 0}
{url.visit_count || 0}
<Actions>
{url.password && <Icon src="/images/lock.svg" lowopacity />}
{url.count > 0 && (
{url.visit_count > 0 && (
<TBodyButton withText onClick={this.goTo}>
<Icon src="/images/chart.svg" />
Stats

+ 9
- 9
client/components/Table/Table.js View File

@ -93,15 +93,15 @@ class Table extends Component {
}, 1500);
}
showModal(e) {
e.preventDefault();
const modalUrlId = e.currentTarget.dataset.id;
const modalUrlDomain = e.currentTarget.dataset.host;
this.setState({
modalUrlId,
modalUrlDomain,
showModal: true,
});
showModal(url) {
return e => {
e.preventDefault();
this.setState({
modalUrlId: url.address,
modalUrlDomain: url.domain,
showModal: true,
});
}
}
closeModal() {

+ 1
- 13
client/pages/_document.js View File

@ -59,20 +59,8 @@ class AppDocument extends Document {
}}
/>
<script
dangerouslySetInnerHTML={{
__html: `
if('serviceWorker' in navigator) {
navigator.serviceWorker.register('sw.js', {
scope: './'
})
}
`,
}}
/>
<script src="https://www.google.com/recaptcha/api.js?render=explicit" async defer />
<script src="/analytics.js" />
<script src="static/analytics.js" />
</Head>
<body style={style}>
<Main />

+ 5
- 3
client/pages/stats.js View File

@ -4,23 +4,25 @@ import BodyWrapper from '../components/BodyWrapper';
import Stats from '../components/Stats';
import { authUser } from '../actions';
const StatsPage = ({ id }) => (
const StatsPage = ({ domain, id }) => (
<BodyWrapper>
<Stats id={id} />
<Stats domain={domain} id={id} />
</BodyWrapper>
);
StatsPage.getInitialProps = ({ req, reduxStore, query }) => {
const token = req && req.cookies && req.cookies.token;
if (token && reduxStore) reduxStore.dispatch(authUser(token));
return { id: query && query.id };
return query;
};
StatsPage.propTypes = {
domain: PropTypes.string,
id: PropTypes.string,
};
StatsPage.defaultProps = {
domain: '',
id: '',
};

+ 114
- 0
global.d.ts View File

@ -0,0 +1,114 @@
interface User {
id: number;
apikey?: string;
banned: boolean;
banned_by_id?: number;
cooldowns?: string[];
created_at: string;
email: string;
password: string;
reset_password_expires?: string;
reset_password_token?: string;
updated_at: string;
verification_expires?: string;
verification_token?: string;
verified?: boolean;
}
interface UserJoined extends User {
admin?: boolean;
homepage?: string;
domain?: string;
domain_id?: number;
}
interface Domain {
id: number;
address: string;
banned: boolean;
banned_by_id?: number;
created_at: string;
homepage?: string;
updated_at: string;
user_id?: number;
}
interface Host {
id: number;
address: string;
banned: boolean;
banned_by_id?: number;
created_at: string;
updated_at: string;
}
interface IP {
id: number;
created_at: string;
updated_at: string;
ip: string;
}
interface Link {
id: number;
address: string;
banned: boolean;
banned_by_id?: number;
created_at: string;
domain_id?: number;
password?: string;
target: string;
updated_at: string;
user_id?: number;
visit_count: number;
}
interface LinkJoinedDomain extends Link {
domain?: string;
}
interface Visit {
id: number;
countries: Record<string, number>;
created_at: string;
link_id: number;
referrers: Record<string, number>;
total: number;
br_chrome: number;
br_edge: number;
br_firefox: number;
br_ie: number;
br_opera: number;
br_other: number;
br_safari: number;
os_android: number;
os_ios: number;
os_linux: number;
os_macos: number;
os_other: number;
os_windows: number;
}
interface Stats {
browser: Record<
'chrome' | 'edge' | 'firefox' | 'ie' | 'opera' | 'other' | 'safari',
number
>;
os: Record<
'android' | 'ios' | 'linux' | 'macos' | 'other' | 'windows',
number
>;
country: Record<string, number>;
referrer: Record<string, number>;
}
declare namespace Express {
export interface Request {
realIP?: string;
pageType?: string;
linkTarget?: string;
protectedLink?: string;
token?: string;
user: UserJoined;
}
}

+ 1
- 1
nodemon.json View File

@ -1,6 +1,6 @@
{
"watch": ["server/**/*.ts"],
"execMap": {
"ts": "rm -rf production-server && tsc --project tsconfig.server.json && cp server/mail/template-reset.html production-server/mail/template-reset.html && cp server/mail/template-verify.html production-server/mail/template-verify.html && node production-server/server.js"
"ts": "rimraf production-server && tsc --project tsconfig.server.json && copyfiles -f \"server/mail/*.html\" production-server/mail && node production-server/server.js"
}
}

+ 1040
- 381
package-lock.json
File diff suppressed because it is too large
View File


+ 28
- 23
package.json View File

@ -8,7 +8,7 @@
"docker:build": "docker build -t kutt .",
"docker:run": "docker run -p 3000:3000 --env-file .env -d kutt:latest",
"dev": "nodemon server/server.ts",
"build": "next build client/ && tsc --project tsconfig.server.json",
"build": "next build client/ && rimraf production-server && tsc --project tsconfig.server.json && copyfiles -f \"server/mail/*.html\" production-server/mail",
"start": "NODE_ENV=production node production-server/server.js",
"lint": "eslint server/ --ext .js,.ts --fix",
"lint:nofix": "eslint server/ --ext .js,.ts"
@ -32,21 +32,6 @@
},
"homepage": "https://github.com/TheDevs-Network/kutt#readme",
"dependencies": {
"@types/body-parser": "^1.17.0",
"@types/cookie-parser": "^1.4.1",
"@types/date-fns": "^2.6.0",
"@types/dotenv": "^4.0.3",
"@types/express": "^4.16.0",
"@types/helmet": "0.0.38",
"@types/jsonwebtoken": "^7.2.8",
"@types/jwt-decode": "^2.2.1",
"@types/mongodb": "^3.1.17",
"@types/mongoose": "^5.3.5",
"@types/next": "^7.0.5",
"@types/passport": "^0.4.7",
"@types/passport-jwt": "^3.0.1",
"@types/redis": "^2.8.10",
"@zeit/next-typescript": "^1.1.1",
"axios": "^0.19.0",
"bcryptjs": "^2.4.3",
"cookie-parser": "^1.4.4",
@ -62,6 +47,7 @@
"js-cookie": "^2.2.0",
"jsonwebtoken": "^8.4.0",
"jwt-decode": "^2.2.0",
"knex": "^0.19.2",
"lodash": "^4.17.11",
"mongoose": "^5.6.4",
"morgan": "^1.9.1",
@ -73,10 +59,13 @@
"next-redux-wrapper": "^2.1.0",
"node-cron": "^2.0.3",
"nodemailer": "^6.3.0",
"p-queue": "^6.1.1",
"passport": "^0.4.0",
"passport-jwt": "^4.0.0",
"passport-local": "^1.0.0",
"passport-localapikey-update": "^0.6.0",
"pg": "^7.12.1",
"pg-query-stream": "^2.0.0",
"prop-types": "^15.7.2",
"qrcode.react": "^0.8.0",
"raven": "^2.6.4",
@ -104,14 +93,28 @@
"@babel/preset-env": "^7.3.1",
"@babel/register": "^7.0.0",
"@types/bcryptjs": "^2.4.2",
"@types/body-parser": "^1.17.0",
"@types/cookie-parser": "^1.4.1",
"@types/cors": "^2.8.5",
"@types/date-fns": "^2.6.0",
"@types/dotenv": "^4.0.3",
"@types/express": "^4.16.0",
"@types/helmet": "0.0.38",
"@types/jsonwebtoken": "^7.2.8",
"@types/jwt-decode": "^2.2.1",
"@types/mongodb": "^3.1.17",
"@types/mongoose": "^5.3.5",
"@types/morgan": "^1.7.36",
"@types/ms": "^0.7.30",
"@types/next": "^7.0.5",
"@types/node-cron": "^2.0.2",
"@types/nodemailer": "^6.2.1",
"@types/passport-local": "^1.0.33",
"@typescript-eslint/eslint-plugin": "^1.13.0",
"@typescript-eslint/parser": "^1.13.0",
"@types/pg": "^7.11.0",
"@types/pg-query-stream": "^1.0.3",
"@types/redis": "^2.8.10",
"@typescript-eslint/eslint-plugin": "^2.0.0",
"@typescript-eslint/parser": "^2.0.0",
"@zeit/next-typescript": "^1.1.1",
"babel": "^6.23.0",
"babel-cli": "^6.26.0",
"babel-core": "^6.26.3",
@ -119,20 +122,22 @@
"babel-plugin-styled-components": "^1.10.0",
"babel-preset-env": "^1.7.0",
"chai": "^4.1.2",
"copyfiles": "^2.1.1",
"deep-freeze": "^0.0.1",
"eslint": "^6.1.0",
"eslint": "^5.4.0",
"eslint-config-airbnb": "^16.1.0",
"eslint-config-prettier": "^6.0.0",
"eslint-config-prettier": "^6.1.0",
"eslint-plugin-import": "^2.16.0",
"eslint-plugin-jsx-a11y": "^6.2.1",
"eslint-plugin-prettier": "^2.7.0",
"eslint-plugin-prettier": "^3.1.0",
"eslint-plugin-react": "^7.14.3",
"husky": "^0.15.0-rc.13",
"mocha": "^5.2.0",
"nock": "^9.3.3",
"nodemon": "^1.18.10",
"prettier": "^1.16.4",
"prettier": "^1.18.2",
"redux-mock-store": "^1.5.3",
"rimraf": "^3.0.0",
"sinon": "^6.0.0",
"typescript": "^3.5.3"
}

+ 0
- 27
scripts/migration.ts View File

@ -1,27 +0,0 @@
// 1. Connect to Neo4j database
// 2. Connect to MongoDB database
// HOSTS
// 1. [NEO4J] Get all hosts
// 2. [MONGODB] Create Hosts
// USERS
// 1. [NEO4J] Get all users
// 2. [MONGODB] Upsert users
// 3. [MONGODB] Update bannedBy
// DOMAINS
// 1. [NEO4J] Get all domains as stream
// 2. [MONGODB] If domain has user, get user
// 3. [MONGODB] Upsert domain
// 4. [MONGODB] Update user to set domain
// LINKS
// 1. [NEO4J] Get all links as stream
// 2. [MONGODB] If link has user and domain, get them
// 3. [MONGODB] Upsert link
// VISISTS
// 1. [NEO4J] For every link get visists as stream
// 2. [JAVaSCRIPT] Sum stats for each visist with the same date
// 3. [MONGODB] Create visits

+ 21
- 21
server/configToEnv.ts View File

@ -1,33 +1,33 @@
/* eslint-disable global-require */
import fs from 'fs';
import path from 'path';
import fs from "fs";
import path from "path";
const hasServerConfig = fs.existsSync(path.resolve(__dirname, 'config.js'));
const hasServerConfig = fs.existsSync(path.resolve(__dirname, "config.js"));
const hasClientConfig = fs.existsSync(
path.resolve(__dirname, '../client/config.js')
path.resolve(__dirname, "../client/config.js")
);
if (hasServerConfig && hasClientConfig) {
const serverConfig = require('./config.js');
const clientConfig = require('../client/config.js');
const serverConfig = require("./config.js");
const clientConfig = require("../client/config.js");
let envTemplate = fs.readFileSync(
path.resolve(__dirname, '../.template.env'),
'utf-8'
path.resolve(__dirname, "../.template.env"),
"utf-8"
);
const configs = {
PORT: serverConfig.PORT || 3000,
DEFAULT_DOMAIN: serverConfig.DEFAULT_DOMAIN || 'localhost:3000',
DB_URI: serverConfig.DB_URI || 'bolt://localhost',
DEFAULT_DOMAIN: serverConfig.DEFAULT_DOMAIN || "localhost:3000",
DB_URI: serverConfig.DB_URI || "bolt://localhost",
DB_USERNAME: serverConfig.DB_USERNAME,
DB_PASSWORD: serverConfig.DB_PASSWORD,
REDIS_DISABLED: serverConfig.REDIS_DISABLED || false,
REDIS_HOST: serverConfig.REDIS_HOST || '127.0.0.1',
REDIS_HOST: serverConfig.REDIS_HOST || "127.0.0.1",
REDIS_PORT: serverConfig.REDIS_PORT || 6379,
REDIS_PASSWORD: serverConfig.REDIS_PASSWORD,
USER_LIMIT_PER_DAY: serverConfig.USER_LIMIT_PER_DAY || 50,
JWT_SECRET: serverConfig.JWT_SECRET || 'securekey',
ADMIN_EMAILS: serverConfig.ADMIN_EMAILS.join(','),
JWT_SECRET: serverConfig.JWT_SECRET || "securekey",
ADMIN_EMAILS: serverConfig.ADMIN_EMAILS.join(","),
RECAPTCHA_SITE_KEY: clientConfig.RECAPTCHA_SITE_KEY,
RECAPTCHA_SECRET_KEY: serverConfig.RECAPTCHA_SECRET_KEY,
GOOGLE_SAFE_BROWSING_KEY: serverConfig.GOOGLE_SAFE_BROWSING_KEY,
@ -40,23 +40,23 @@ if (hasServerConfig && hasClientConfig) {
MAIL_FROM: serverConfig.MAIL_FROM,
MAIL_PASSWORD: serverConfig.MAIL_PASSWORD,
REPORT_MAIL: serverConfig.REPORT_MAIL,
CONTACT_EMAIL: clientConfig.CONTACT_EMAIL,
CONTACT_EMAIL: clientConfig.CONTACT_EMAIL
};
Object.keys(configs).forEach(c => {
envTemplate = envTemplate.replace(
new RegExp(`{{${c}}}`, 'gm'),
configs[c] || ''
new RegExp(`{{${c}}}`, "gm"),
configs[c] || ""
);
});
fs.writeFileSync(path.resolve(__dirname, '../.env'), envTemplate);
fs.writeFileSync(path.resolve(__dirname, "../.env"), envTemplate);
fs.renameSync(
path.resolve(__dirname, 'config.js'),
path.resolve(__dirname, 'old.config.js')
path.resolve(__dirname, "config.js"),
path.resolve(__dirname, "old.config.js")
);
fs.renameSync(
path.resolve(__dirname, '../client/config.js'),
path.resolve(__dirname, '../client/old.config.js')
path.resolve(__dirname, "../client/config.js"),
path.resolve(__dirname, "../client/old.config.js")
);
}

+ 87
- 86
server/controllers/authController.ts View File

@ -1,13 +1,14 @@
import { RequestHandler } from 'express';
import fs from 'fs';
import path from 'path';
import passport from 'passport';
import JWT from 'jsonwebtoken';
import axios from 'axios';
import { isAdmin } from '../utils';
import transporter from '../mail/mail';
import { resetMailText, verifyMailText } from '../mail/text';
import { Handler } from "express";
import fs from "fs";
import path from "path";
import passport from "passport";
import JWT from "jsonwebtoken";
import axios from "axios";
import { addDays } from "date-fns";
import { isAdmin } from "../utils";
import transporter from "../mail/mail";
import { resetMailText, verifyMailText } from "../mail/text";
import {
createUser,
changePassword,
@ -15,45 +16,44 @@ import {
getUser,
verifyUser,
requestPasswordReset,
resetPassword,
} from '../db/user';
import { IUser } from '../models/user';
resetPassword
} from "../db/user";
/* Read email template */
const resetEmailTemplatePath = path.join(
__dirname,
'../mail/template-reset.html'
"../mail/template-reset.html"
);
const verifyEmailTemplatePath = path.join(
__dirname,
'../mail/template-verify.html'
"../mail/template-verify.html"
);
const resetEmailTemplate = fs
.readFileSync(resetEmailTemplatePath, { encoding: 'utf-8' })
.readFileSync(resetEmailTemplatePath, { encoding: "utf-8" })
.replace(/{{domain}}/gm, process.env.DEFAULT_DOMAIN);
const verifyEmailTemplate = fs
.readFileSync(verifyEmailTemplatePath, { encoding: 'utf-8' })
.readFileSync(verifyEmailTemplatePath, { encoding: "utf-8" })
.replace(/{{domain}}/gm, process.env.DEFAULT_DOMAIN);
/* Function to generate JWT */
const signToken = (user: IUser) =>
const signToken = (user: UserJoined) =>
JWT.sign(
{
iss: 'ApiAuth',
sub: () => user.email,
domain: (user.domain && user.domain.name) || '',
iss: "ApiAuth",
sub: user.email,
domain: user.domain || "",
admin: isAdmin(user.email),
iat: new Date().getTime(),
exp: new Date().setDate(new Date().getDate() + 7),
},
iat: parseInt((new Date().getTime() / 1000).toFixed(0)),
exp: parseInt((addDays(new Date(), 7).getTime() / 1000).toFixed(0))
} as Record<string, any>,
process.env.JWT_SECRET
);
/* Passport.js authentication controller */
const authenticate = (
type: 'jwt' | 'local' | 'localapikey',
type: "jwt" | "local" | "localapikey",
error: string,
isStrict: boolean = true
isStrict = true
) =>
function auth(req, res, next) {
if (req.user) return next();
@ -63,19 +63,19 @@ const authenticate = (
if (user && isStrict && !user.verified) {
return res.status(400).json({
error:
'Your email address is not verified.' +
'Click on signup to get the verification link again.',
"Your email address is not verified. " +
"Click on signup to get the verification link again."
});
}
if (user && user.banned) {
return res
.status(400)
.json({ error: 'Your are banned from using this website.' });
.json({ error: "Your are banned from using this website." });
}
if (user) {
req.user = {
...user,
admin: isAdmin(user.email),
admin: isAdmin(user.email)
};
return next();
}
@ -84,84 +84,85 @@ const authenticate = (
};
export const authLocal = authenticate(
'local',
'Login email and/or password are wrong.'
"local",
"Login email and/or password are wrong."
);
export const authJwt = authenticate('jwt', 'Unauthorized.');
export const authJwtLoose = authenticate('jwt', 'Unauthorized.', false);
export const authJwt = authenticate("jwt", "Unauthorized.");
export const authJwtLoose = authenticate("jwt", "Unauthorized.", false);
export const authApikey = authenticate(
'localapikey',
'API key is not correct.',
"localapikey",
"API key is not correct.",
false
);
/* reCaptcha controller */
export const recaptcha: RequestHandler = async (req, res, next) => {
if (process.env.NODE_ENV === 'production' && !req.user) {
export const recaptcha: Handler = async (req, res, next) => {
if (process.env.NODE_ENV === "production" && !req.user) {
const isReCaptchaValid = await axios({
method: 'post',
url: 'https://www.google.com/recaptcha/api/siteverify',
method: "post",
url: "https://www.google.com/recaptcha/api/siteverify",
headers: {
'Content-type': 'application/x-www-form-urlencoded',
"Content-type": "application/x-www-form-urlencoded"
},
params: {
secret: process.env.RECAPTCHA_SECRET_KEY,
response: req.body.reCaptchaToken,
remoteip: req.realIP,
},
remoteip: req.realIP
}
});
if (!isReCaptchaValid.data.success) {
return res
.status(401)
.json({ error: 'reCAPTCHA is not valid. Try again.' });
.json({ error: "reCAPTCHA is not valid. Try again." });
}
}
return next();
};
export const authAdmin: RequestHandler = async (req, res, next) => {
export const authAdmin: Handler = async (req, res, next) => {
if (!req.user.admin) {
return res.status(401).json({ error: 'Unauthorized.' });
return res.status(401).json({ error: "Unauthorized." });
}
return next();
};
export const signup: RequestHandler = async (req, res) => {
export const signup: Handler = async (req, res) => {
const { email, password } = req.body;
if (password.length > 64) {
return res.status(400).json({ error: 'Maximum password length is 64.' });
return res.status(400).json({ error: "Maximum password length is 64." });
}
if (email.length > 64) {
return res.status(400).json({ error: 'Maximum email length is 64.' });
if (email.length > 255) {
return res.status(400).json({ error: "Maximum email length is 255." });
}
const user = await getUser(email);
if (user && user.verified)
return res.status(403).json({ error: 'Email is already in use.' });
if (user && user.verified) {
return res.status(403).json({ error: "Email is already in use." });
}
const newUser = await createUser(email, password);
const newUser = await createUser(email, password, user);
const mail = await transporter.sendMail({
from: process.env.MAIL_FROM || process.env.MAIL_USER,
to: newUser.email,
subject: 'Verify your account',
subject: "Verify your account",
text: verifyMailText.replace(
/{{verification}}/gim,
newUser.verificationToken
newUser.verification_token
),
html: verifyEmailTemplate.replace(
/{{verification}}/gim,
newUser.verificationToken
),
newUser.verification_token
)
});
if (mail.accepted.length) {
return res
.status(201)
.json({ email, message: 'Verification email has been sent.' });
.json({ email, message: "Verification email has been sent." });
}
return res
@ -169,42 +170,42 @@ export const signup: RequestHandler = async (req, res) => {
.json({ error: "Couldn't send verification email. Try again." });
};
export const login: RequestHandler = (req, res) => {
export const login: Handler = (req, res) => {
const token = signToken(req.user);
return res.status(200).json({ token });
};
export const renew: RequestHandler = (req, res) => {
export const renew: Handler = (req, res) => {
const token = signToken(req.user);
return res.status(200).json({ token });
};
export const verify: RequestHandler = async (req, _res, next) => {
export const verify: Handler = async (req, _res, next) => {
const user = await verifyUser(req.params.verificationToken);
if (user) {
const token = signToken(user);
req.user = { token };
req.token = token;
}
return next();
};
export const changeUserPassword: RequestHandler = async (req, res) => {
export const changeUserPassword: Handler = async (req, res) => {
if (req.body.password.length < 8) {
return res
.status(400)
.json({ error: 'Password must be at least 8 chars long.' });
.json({ error: "Password must be at least 8 chars long." });
}
if (req.body.password.length > 64) {
return res.status(400).json({ error: 'Maximum password length is 64.' });
return res.status(400).json({ error: "Maximum password length is 64." });
}
const changedUser = await changePassword(req.user._id, req.body.password);
const changedUser = await changePassword(req.user.id, req.body.password);
if (changedUser) {
return res
.status(200)
.json({ message: 'Your password has been changed successfully.' });
.json({ message: "Your password has been changed successfully." });
}
return res
@ -212,26 +213,26 @@ export const changeUserPassword: RequestHandler = async (req, res) => {
.json({ error: "Couldn't change the password. Try again later" });
};
export const generateUserApiKey: RequestHandler = async (req, res) => {
const user = await generateApiKey(req.user._id);
export const generateUserApiKey = async (req, res) => {
const apikey = await generateApiKey(req.user.id);
if (user.apikey) {
return res.status(201).json({ apikey: user.apikey });
if (apikey) {
return res.status(201).json({ apikey });
}
return res
.status(400)
.json({ error: 'Sorry, an error occured. Please try again later.' });
.json({ error: "Sorry, an error occured. Please try again later." });
};
export const userSettings: RequestHandler = (req, res) =>
export const userSettings: Handler = (req, res) =>
res.status(200).json({
apikey: req.user.apikey || '',
customDomain: req.user.domain || '',
homepage: req.user.homepage || '',
apikey: req.user.apikey || "",
customDomain: req.user.domain || "",
homepage: req.user.homepage || ""
});
export const requestUserPasswordReset: RequestHandler = async (req, res) => {
export const requestUserPasswordReset: Handler = async (req, res) => {
const user = await requestPasswordReset(req.body.email);
if (!user) {
@ -241,30 +242,30 @@ export const requestUserPasswordReset: RequestHandler = async (req, res) => {
const mail = await transporter.sendMail({
from: process.env.MAIL_USER,
to: user.email,
subject: 'Reset your password',
subject: "Reset your password",
text: resetMailText
.replace(/{{resetpassword}}/gm, user.resetPasswordToken)
.replace(/{{resetpassword}}/gm, user.reset_password_token)
.replace(/{{domain}}/gm, process.env.DEFAULT_DOMAIN),
html: resetEmailTemplate
.replace(/{{resetpassword}}/gm, user.resetPasswordToken)
.replace(/{{domain}}/gm, process.env.DEFAULT_DOMAIN),
.replace(/{{resetpassword}}/gm, user.reset_password_token)
.replace(/{{domain}}/gm, process.env.DEFAULT_DOMAIN)
});
if (mail.accepted.length) {
return res.status(200).json({
email: user.email,
message: 'Reset password email has been sent.',
message: "Reset password email has been sent."
});
}
return res.status(400).json({ error: "Couldn't reset password." });
};
export const resetUserPassword: RequestHandler = async (req, _res, next) => {
const user = await resetPassword(req.params.resetPasswordToken);
export const resetUserPassword: Handler = async (req, _res, next) => {
const user: UserJoined = await resetPassword(req.params.resetPasswordToken);
if (user) {
const token = signToken(user);
req.user = { token };
const token = signToken(user as UserJoined);
req.token = token;
}
return next();
};

+ 160
- 164
server/controllers/linkController.ts View File

@ -1,123 +1,124 @@
import { Handler } from 'express';
import { promisify } from 'util';
import urlRegex from 'url-regex';
import dns from 'dns';
import URL from 'url';
import generate from 'nanoid/generate';
import useragent from 'useragent';
import geoip from 'geoip-lite';
import bcrypt from 'bcryptjs';
import ua from 'universal-analytics';
import isbot from 'isbot';
import { addIP } from '../db/ip';
import bcrypt from "bcryptjs";
import dns from "dns";
import { Handler } from "express";
import geoip from "geoip-lite";
import isbot from "isbot";
import generate from "nanoid/generate";
import ua from "universal-analytics";
import URL from "url";
import urlRegex from "url-regex";
import useragent from "useragent";
import { promisify } from "util";
import { deleteDomain, getDomain, setDomain } from "../db/domain";
import { addIP } from "../db/ip";
import {
addLinkCount,
banLink,
createShortLink,
createVisit,
deleteLink,
findLink,
getUserLinksCount,
getStats,
getLinks,
banLink,
} from '../db/link';
getStats,
getUserLinksCount
} from "../db/link";
import transporter from "../mail/mail";
import * as redis from "../redis";
import {
addProtocol,
generateShortLink,
getStatsCacheTime,
getStatsLimit
} from "../utils";
import {
checkBannedDomain,
checkBannedHost,
cooldownCheck,
malwareCheck,
preservedUrls,
urlCountsCheck,
} from './validateBodyController';
import { getDomain, setDomain, deleteDomain } from '../db/domain';
import transporter from '../mail/mail';
import * as redis from '../redis';
import {
addProtocol,
getStatsLimit,
generateShortLink,
getStatsCacheTime,
} from '../utils';
import { IDomain } from '../models/domain';
urlCountsCheck
} from "./validateBodyController";
const dnsLookup = promisify(dns.lookup);
const generateId = async () => {
const id = generate(
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890',
const address = generate(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890",
6
);
const link = await findLink({ id });
if (!link) return id;
const link = await findLink({ address });
if (!link) return address;
return generateId();
};
export const shortener: Handler = async (req, res) => {
try {
const targetDomain = URL.parse(req.body.target).hostname;
const target = addProtocol(req.body.target);
const targetDomain = URL.parse(target).hostname;
const queries = await Promise.all([
process.env.GOOGLE_SAFE_BROWSING_KEY && cooldownCheck(req.user),
process.env.GOOGLE_SAFE_BROWSING_KEY &&
malwareCheck(req.user, req.body.target),
req.user && urlCountsCheck(req.user._id),
req.user && urlCountsCheck(req.user),
req.user &&
req.body.reuse &&
findLink({ target: addProtocol(req.body.target), user: req.user._id }),
findLink({
target,
user_id: req.user.id
}),
req.user &&
req.body.customurl &&
findLink(
{
id: req.body.customurl,
domain: req.user.domain && req.user.domain._id,
},
{ forceDomainCheck: true }
),
findLink({
address: req.body.customurl,
domain_id: req.user.domain_id || null
}),
(!req.user || !req.body.customurl) && generateId(),
checkBannedDomain(targetDomain),
checkBannedHost(targetDomain),
checkBannedHost(targetDomain)
]);
// if "reuse" is true, try to return
// the existent URL without creating one
if (queries[3]) {
const { domain: d, user: u, ...link } = queries[3];
const { domain_id: d, user_id: u, ...link } = queries[3];
const data = {
...link,
id: link.address,
password: !!link.password,
reuse: true,
shortLink: generateShortLink(link.id, req.user.domain),
shortLink: generateShortLink(link.address, req.user.domain)
};
return res.json(data);
}
// Check if custom link already exists
if (queries[4]) {
throw new Error('Custom URL is already in use.');
throw new Error("Custom URL is already in use.");
}
// Create new link
const id = (req.user && req.body.customurl) || queries[5];
const target = addProtocol(req.body.target);
const link = await createShortLink({
...req.body,
id,
target,
user: req.user,
});
const address = (req.user && req.body.customurl) || queries[5];
const link = await createShortLink(
{
...req.body,
address,
target
},
req.user
);
if (!req.user && Number(process.env.NON_USER_COOLDOWN)) {
addIP(req.realIP);
}
return res.json(link);
return res.json({ ...link, id: link.address });
} catch (error) {
return res.status(400).json({ error: error.message });
}
};
const browsersList = ['IE', 'Firefox', 'Chrome', 'Opera', 'Safari', 'Edge'];
const osList = ['Windows', 'Mac Os', 'Linux', 'Android', 'iOS'];
const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
const filterInBrowser = agent => item =>
agent.family.toLowerCase().includes(item.toLocaleLowerCase());
const filterInOs = agent => item =>
@ -126,20 +127,21 @@ const filterInOs = agent => item =>
export const goToLink: Handler = async (req, res, next) => {
const { host } = req.headers;
const reqestedId = req.params.id || req.body.id;
const id = reqestedId.replace('+', '');
const address = reqestedId.replace("+", "");
const customDomain = host !== process.env.DEFAULT_DOMAIN && host;
const agent = useragent.parse(req.headers['user-agent']);
const [browser = 'Other'] = browsersList.filter(filterInBrowser(agent));
const [os = 'Other'] = osList.filter(filterInOs(agent));
// TODO: Extract parsing into their own function
const agent = useragent.parse(req.headers["user-agent"]);
const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
const [os = "Other"] = osList.filter(filterInOs(agent));
const referrer =
req.header('Referer') && URL.parse(req.header('Referer')).hostname;
req.header("Referer") && URL.parse(req.header("Referer")).hostname;
const location = geoip.lookup(req.realIP);
const country = location && location.country;
const isBot = isbot(req.headers['user-agent']);
const isBot = isbot(req.headers["user-agent"]);
const domain = await (customDomain && getDomain({ name: customDomain }));
const domain = await (customDomain && getDomain({ address: customDomain }));
const link = await findLink({ id, domain: domain && domain._id });
const link = await findLink({ address, domain_id: domain && domain.id });
if (!link) {
if (host !== process.env.DEFAULT_DOMAIN) {
@ -150,51 +152,51 @@ export const goToLink: Handler = async (req, res, next) => {
}
if (link.banned) {
return res.redirect('/banned');
return res.redirect("/banned");
}
const doesRequestInfo = /.*\+$/gi.test(reqestedId);
if (doesRequestInfo && !link.password) {
req.linkTarget = link.target;
req.pageType = 'info';
req.pageType = "info";
return next();
}
if (link.password && !req.body.password) {
req.protectedLink = id;
req.pageType = 'password';
req.protectedLink = address;
req.pageType = "password";
return next();
}
if (link.password) {
const isMatch = await bcrypt.compare(req.body.password, link.password);
if (!isMatch) {
return res.status(401).json({ error: 'Password is not correct' });
return res.status(401).json({ error: "Password is not correct" });
}
if (link.user && !isBot) {
addLinkCount(link.id, customDomain);
if (link.user_id && !isBot) {
addLinkCount(link.id);
createVisit({
browser: browser.toLowerCase(),
country: country || 'Unknown',
country: country || "Unknown",
domain: customDomain,
id: link.id,
os: os.toLowerCase().replace(/\s/gi, ''),
referrer: referrer.replace(/\./gi, '[dot]') || 'Direct',
limit: getStatsLimit(),
os: os.toLowerCase().replace(/\s/gi, ""),
referrer: referrer.replace(/\./gi, "[dot]") || "Direct",
limit: getStatsLimit()
});
}
return res.status(200).json({ target: link.target });
}
if (link.user && !isBot) {
addLinkCount(link.id, customDomain);
if (link.user_id && !isBot) {
addLinkCount(link.id);
createVisit({
browser,
country: country || 'Unknown',
browser: browser.toLowerCase(),
country: (country && country.toLocaleLowerCase()) || "unknown",
domain: customDomain,
id: link.id,
os,
referrer: referrer || 'Direct',
limit: getStatsLimit(),
os: os.toLowerCase().replace(/\s/gi, ""),
referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "direct",
limit: getStatsLimit()
});
}
@ -202,10 +204,10 @@ export const goToLink: Handler = async (req, res, next) => {
const visitor = ua(process.env.GOOGLE_ANALYTICS_UNIVERSAL);
visitor
.pageview({
dp: `/${id}`,
ua: req.headers['user-agent'],
dp: `/${address}`,
ua: req.headers["user-agent"],
uip: req.realIP,
aip: 1,
aip: 1
})
.send();
}
@ -216,8 +218,8 @@ export const goToLink: Handler = async (req, res, next) => {
export const getUserLinks: Handler = async (req, res) => {
// TODO: Use aggregation
const [countAll, list] = await Promise.all([
getUserLinksCount({ user: req.user }),
getLinks(req.user, req.query),
getUserLinksCount({ user_id: req.user.id }),
getLinks(req.user.id, req.query)
]);
return res.json({ list, countAll });
};
@ -226,11 +228,11 @@ export const setCustomDomain: Handler = async (req, res) => {
const parsed = URL.parse(req.body.customDomain);
const customDomain = parsed.hostname || parsed.href;
if (!customDomain)
return res.status(400).json({ error: 'Domain is not valid.' });
return res.status(400).json({ error: "Domain is not valid." });
if (customDomain.length > 40) {
return res
.status(400)
.json({ error: 'Maximum custom domain length is 40.' });
.json({ error: "Maximum custom domain length is 40." });
}
if (customDomain === process.env.DEFAULT_DOMAIN) {
return res.status(400).json({ error: "You can't use default domain." });
@ -239,30 +241,34 @@ export const setCustomDomain: Handler = async (req, res) => {
!req.body.homepage ||
urlRegex({ exact: true, strict: false }).test(req.body.homepage);
if (!isValidHomepage)
return res.status(400).json({ error: 'Homepage is not valid.' });
return res.status(400).json({ error: "Homepage is not valid." });
const homepage =
req.body.homepage &&
(URL.parse(req.body.homepage).protocol
? req.body.homepage
: `http://${req.body.homepage}`);
const matchedDomain = await getDomain({ name: customDomain });
const matchedDomain = await getDomain({ address: customDomain });
if (
matchedDomain &&
matchedDomain.user.toString() !== req.user._id.toString()
matchedDomain.user_id &&
matchedDomain.user_id !== req.user.id
) {
return res.status(400).json({
error: 'Domain is already taken. Contact us for multiple users.',
error: "Domain is already taken. Contact us for multiple users."
});
}
const userCustomDomain = await setDomain({
user: req.user,
name: customDomain,
homepage,
});
const userCustomDomain = await setDomain(
{
address: customDomain,
homepage
},
req.user,
matchedDomain
);
if (userCustomDomain) {
return res.status(201).json({
customDomain: userCustomDomain.name,
homepage: userCustomDomain.homepage,
customDomain: userCustomDomain.address,
homepage: userCustomDomain.homepage
});
}
return res.status(400).json({ error: "Couldn't set custom domain." });
@ -271,7 +277,7 @@ export const setCustomDomain: Handler = async (req, res) => {
export const deleteCustomDomain: Handler = async (req, res) => {
const response = await deleteDomain(req.user);
if (response)
return res.status(200).json({ message: 'Domain deleted successfully' });
return res.status(200).json({ message: "Domain deleted successfully" });
return res.status(400).json({ error: "Couldn't delete custom domain." });
};
@ -279,12 +285,12 @@ export const customDomainRedirection: Handler = async (req, res, next) => {
const { headers, path } = req;
if (
headers.host !== process.env.DEFAULT_DOMAIN &&
(path === '/' ||
(path === "/" ||
preservedUrls
.filter(l => l !== 'url-password')
.some(item => item === path.replace('/', '')))
.filter(l => l !== "url-password")
.some(item => item === path.replace("/", "")))
) {
const domain = await getDomain({ name: headers.host });
const domain = await getDomain({ address: headers.host });
return res.redirect(
301,
(domain && domain.homepage) ||
@ -295,92 +301,82 @@ export const customDomainRedirection: Handler = async (req, res, next) => {
};
export const deleteUserLink: Handler = async (req, res) => {
if (!req.body.id)
return res.status(400).json({ error: 'No id has been provided.' });
const customDomain =
req.body.domain !== process.env.DEFAULT_DOMAIN
? await getDomain({ name: req.body.domain })
: ({} as IDomain);
const link = await findLink(
{
id: req.body.id,
domain: customDomain._id,
user: req.user && req.user._id,
},
{ forceDomainCheck: true }
);
if (!link) {
return res.status(400).json({ error: "Couldn't find the short link." });
const { id, domain } = req.body;
if (!id) {
return res.status(400).json({ error: "No id has been provided." });
}
const response = await deleteLink({
id: req.body,
domain: customDomain._id,
user: req.user,
address: id,
domain: domain !== process.env.DEFAULT_DOMAIN && domain,
user_id: req.user.id
});
if (response) {
return res.status(200).json({ message: 'Short link deleted successfully' });
return res.status(200).json({ message: "Short link deleted successfully" });
}
return res.status(400).json({ error: "Couldn't delete the short link." });
};
export const getLinkStats: Handler = async (req, res) => {
if (!req.query.id)
return res.status(400).json({ error: 'No id has been provided.' });
const customDomain =
req.query.domain !== process.env.DEFAULT_DOMAIN
? await getDomain({ name: req.query.domain })
: ({} as IDomain);
const redisKey = req.query.id + (customDomain.name || '') + req.user.email;
if (!req.query.id) {
return res.status(400).json({ error: "No id has been provided." });
}
const { hostname } = URL.parse(req.query.domain);
const hasCustomDomain =
req.query.domain && hostname !== process.env.DEFAULT_DOMAIN;
const customDomain = hasCustomDomain
? (await getDomain({ address: req.query.domain })) || ({ id: -1 } as Domain)
: ({} as Domain);
const redisKey = req.query.id + (customDomain.address || "") + req.user.email;
const cached = await redis.get(redisKey);
if (cached) return res.status(200).json(JSON.parse(cached));
const link = await findLink(
{
id: req.query.id,
domain: customDomain._id,
user: req.user && req.user._id,
},
{ forceDomainCheck: true }
);
if (!link)
return res.status(400).json({ error: "Couldn't find the short link." });
const stats = await getStats({
id: req.query.id,
domain: customDomain._id,
user: req.user,
const link = await findLink({
address: req.query.id,
domain_id: hasCustomDomain ? customDomain.id : null,
user_id: req.user && req.user.id
});
if (!link) {
return res.status(400).json({ error: "Couldn't find the short link." });
}
const stats = await getStats(link, customDomain);
if (!stats) {
return res
.status(400)
.json({ error: 'Could not get the short link stats.' });
.json({ error: "Could not get the short link stats." });
}
const cacheTime = getStatsCacheTime(stats.total);
redis.set(redisKey, JSON.stringify(stats), 'EX', cacheTime);
const cacheTime = getStatsCacheTime(0);
redis.set(redisKey, JSON.stringify(stats), "EX", cacheTime);
return res.status(200).json(stats);
};
export const reportLink: Handler = async (req, res) => {
// TODO: Change from url to link in front-end
if (!req.body.link)
return res.status(400).json({ error: 'No URL has been provided.' });
if (!req.body.link) {
return res.status(400).json({ error: "No URL has been provided." });
}
const { hostname } = URL.parse(req.body.link);
if (hostname !== process.env.DEFAULT_DOMAIN) {
return res.status(400).json({
error: `You can only report a ${process.env.DEFAULT_DOMAIN} link`,
error: `You can only report a ${process.env.DEFAULT_DOMAIN} link`
});
}
const mail = await transporter.sendMail({
from: process.env.MAIL_USER,
to: process.env.REPORT_MAIL,
subject: '[REPORT]',
subject: "[REPORT]",
text: req.body.url,
html: req.body.url,
html: req.body.url
});
if (mail.accepted.length) {
return res
@ -394,14 +390,14 @@ export const reportLink: Handler = async (req, res) => {
export const ban: Handler = async (req, res) => {
if (!req.body.id)
return res.status(400).json({ error: 'No id has been provided.' });
return res.status(400).json({ error: "No id has been provided." });
const link = await findLink({ id: req.body.id }, { forceDomainCheck: true });
const link = await findLink({ address: req.body.id, domain_id: null });
if (!link) return res.status(400).json({ error: "Couldn't find the link." });
if (link.banned) {
return res.status(200).json({ message: 'Link was banned already' });
return res.status(200).json({ message: "Link was banned already." });
}
const domain = URL.parse(link.target).hostname;
@ -417,12 +413,12 @@ export const ban: Handler = async (req, res) => {
}
await banLink({
adminId: req.user,
adminId: req.user.id,
domain,
host,
id: req.body.id,
banUser: !!req.body.user,
address: req.body.id,
banUser: !!req.body.user
});
return res.status(200).json({ message: 'Link has been banned successfully' });
return res.status(200).json({ message: "Link has been banned successfully" });
};

+ 80
- 85
server/controllers/validateBodyController.ts View File

@ -1,37 +1,36 @@
import { RequestHandler } from 'express';
import { promisify } from 'util';
import dns from 'dns';
import axios from 'axios';
import URL from 'url';
import urlRegex from 'url-regex';
import validator from 'express-validator/check';
import { differenceInMinutes, subHours, subDays } from 'date-fns';
import { validationResult } from 'express-validator/check';
import { IUser } from '../models/user';
import { addCooldown, banUser } from '../db/user';
import { getIP } from '../db/ip';
import { getUserLinksCount } from '../db/link';
import { getDomain } from '../db/domain';
import { getHost } from '../db/host';
import { addProtocol } from '../utils';
import { RequestHandler } from "express";
import { promisify } from "util";
import dns from "dns";