Compare commits

11 Commits
main ... dev

Author SHA1 Message Date
ccec4cdf27 Fixed looped dependencies and updated logging 2025-08-31 22:31:07 +03:00
4d4fce186c changed ping endpoints for better frontend support 2025-08-27 18:19:24 +03:00
aeed5ea45c imported routers to fastapi app routers 2025-08-07 18:57:41 +03:00
10310b9416 updated notes 2025-08-06 19:00:51 +03:00
f9277ad570 implemented most of the pixiv scraper 2025-08-06 19:00:19 +03:00
3a25f4fdd4 starting scraper development 2025-08-05 19:14:24 +03:00
a5c512c7d4 idk WIP2 2025-08-05 12:10:56 +03:00
18b13fdb57 idk WIP 2025-08-04 20:43:47 +03:00
d5ed160746 fixed non existing users column 2025-08-04 10:22:44 +03:00
f9cfa1648e feed generation and accounts WIP 2025-08-01 18:31:55 +03:00
8ba9b7816e feed WIP 2025-07-31 20:15:48 +03:00
30 changed files with 1143 additions and 160 deletions

View File

@ -1,17 +1,9 @@
# Используем официальный образ Python 3
FROM python:3.13-slim
# Устанавливаем рабочую директорию внутри контейнера
WORKDIR /app
# Копируем requirements.txt в контейнер для установки зависимостей
COPY requirements.txt ./
# Устанавливаем зависимости
RUN pip install --no-cache-dir -r requirements.txt
# Копируем остальные файлы проекта в контейнер
COPY . .
# Определяем команду запуска приложения
CMD ["python", "src/__main__.py"]

7
notes.md Normal file
View File

@ -0,0 +1,7 @@
### Сомнения по API:
1) `feeds` API delete feed
2) Нужно ли API картинок? Вроде вообще всё будет делать scraper. (хотя наверное получение инфы о картинке по id - нужная штука)
3) Возможно стоит добавить возможность динамической смены длины invite кода
### TODO:
1) Реализовать лайк картинки на платформе, если её лайкнуло больше n/2 (предварительно) участников группы. Так же что-то придумать с АФК челами, которые будут не давать ленте меняться, своим присутствием и не-голосованием. Возможно сделать систему очков, типа +1 за лайк, -1 за дизлайк и сохранять только перед генерацией новой ленты? (звучит как упор в rate limit и вообще..)

Binary file not shown.

100
src/api/accounts.py Normal file
View File

@ -0,0 +1,100 @@
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, status
from psycopg2._psycopg import connection
import db.accounts as db
import settings.settings as settings
from api.models import Account, User
from api.string_tools import encode_str
from api.utils import get_current_user
from db.groups import check_group_author
from db.internal import get_db_connection
accounts_router = APIRouter(prefix="/api/accounts", tags=["accounts"])
@accounts_router.post("/group")
async def read_accounts_by_group(
groupname: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if not check_group_author(conn, groupname, current_user.username) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return db.get_accounts_by_group(conn, groupname)
@accounts_router.post("/group/platform")
async def read_accounts_by_group_platform(
groupname: str,
platform: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
account_data = db.get_accounts_by_group_platform(conn, groupname, platform.lower())
if account_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such account",
)
return Account().fill(account_data)
@accounts_router.post("/add")
async def add_account(
groupname: str,
platform: str,
login: str,
password: str,
metadata: dict,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if not check_group_author(conn, groupname, current_user.username) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
if db.check_account_existence(conn, groupname, platform.lower()):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Account already exists",
)
hashed_password = encode_str(password)
return db.create_account(conn, groupname, current_user.username, platform.lower(), login, hashed_password, metadata)
@accounts_router.post("/update")
async def update_account(
groupname: str,
platform: str,
author: str,
login: str,
password: str,
metadata: dict,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
account_data = db.get_accounts_by_group_platform(conn, groupname, platform.lower())
if account_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such account",
)
account = Account().fill(account_data)
if current_user.username != account.author and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return db.update_account(conn, groupname, author, platform.lower(), login, password, metadata)

View File

@ -0,0 +1,87 @@
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, status
from psycopg2._psycopg import connection
import db.feeds as db
import settings.settings as settings
from api.models import Feed, User
from api.utils import get_current_user
from db.accounts import get_accounts_by_group
from db.feeds import get_groupname_by_feed_id
from db.groups import check_group_author
from db.internal import get_db_connection
from db.memberships import check_membership_exists
from scraper.utils import generate_feed
feeds_router = APIRouter(prefix="/api/feeds", tags=["feeds"])
@feeds_router.post("/feed")
async def read_feed(
feed_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
feed_data = db.get_feed(conn, feed_id)
if feed_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed",
)
feed = Feed().fill(feed_data)
groupname = get_groupname_by_feed_id(conn, feed_id)
if groupname is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed",
)
if not check_membership_exists(conn, current_user.username, groupname) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return feed
# TODO: review exception
@feeds_router.post("/new")
async def new_feed(
groupname: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if not check_group_author(conn, groupname, current_user.username) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
accounts = get_accounts_by_group(conn, groupname)
feed = generate_feed(conn, accounts)
if feed:
return db.create_feed(conn, groupname, feed)
else:
raise HTTPException(
status_code=status.HTTP_424_FAILED_DEPENDENCY,
detail="Failed to generate feed",
)
# maybe to delete
@feeds_router.post("/delete")
async def delete_feed(
feed_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if current_user.role in settings.settings.admin_roles:
return db.delete_feed(conn, feed_id)
else:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)

View File

@ -5,13 +5,23 @@ from fastapi import APIRouter, Depends, HTTPException, status
import settings.settings as settings
from api.models import User
from api.utils import get_current_user
from settings.consts import API_EDITABLE_SETTINGS_LIST
from settings.settings import load_settings, reset_settings, save_settings
general_router = APIRouter(prefix="/api", tags=["general"])
@general_router.get("/ping")
async def ping():
@general_router.get("/ping/user")
async def ping_user(current_user: Annotated[User, Depends(get_current_user)]):
return {"ok"}
@general_router.get("/ping/admin")
async def ping_admin(current_user: Annotated[User, Depends(get_current_user)]):
if current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return {"ok"}
@ -25,7 +35,7 @@ async def get_settings(current_user: Annotated[User, Depends(get_current_user)])
return settings.settings
@general_router.post("/settings/update")
@general_router.post("/settings/update", description=API_EDITABLE_SETTINGS_LIST)
async def update_settings(data: dict, current_user: Annotated[User, Depends(get_current_user)]):
if current_user.role not in settings.settings.admin_roles:
raise HTTPException(

View File

@ -8,7 +8,7 @@ import db.groups as db
import settings.settings as settings
import settings.startup_settings as startup_settings
from api.models import Group, User
from api.utils import get_current_user, get_group_by_name
from api.utils import get_current_user
from db.internal import get_db_connection
from db.memberships import check_membership_exists
from settings.consts import JOIN_CODE_SYMBOLS
@ -27,15 +27,13 @@ async def read_any_group(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
group = Group()
group_data = db.get_group(conn, groupname)
if group_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such group",
)
group.fill(group_data)
return group
return Group().fill(group_data)
@groups_router.post("/invite_code")
async def read_group_invite_code(
@ -56,6 +54,19 @@ async def read_group_invite_code(
)
return invite_code
@groups_router.post("/last_feed")
async def read_group_last_feed_id(
groupname: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if not check_membership_exists(conn, current_user.username, groupname) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return db.get_group_last_feed_id(conn, groupname)
@groups_router.post("/add")
async def add_group(
@ -91,10 +102,9 @@ async def delete_group(
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
group = get_group_by_name(conn, groupname)
if current_user.role in settings.settings.admin_roles:
return db.delete_group(conn, groupname)
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return db.delete_group(conn, groupname)
else:
raise HTTPException(
@ -115,11 +125,10 @@ async def update_groupname(
status_code=status.HTTP_409_CONFLICT,
detail="Groupname is already taken",
)
group = get_group_by_name(conn, groupname)
if current_user.role in settings.settings.admin_roles:
return db.update_group_groupname(conn, groupname, new_groupname)
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return db.update_group_groupname(conn, groupname, new_groupname)
else:
raise HTTPException(
@ -134,10 +143,9 @@ async def update_author(
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
group = get_group_by_name(conn, groupname)
if current_user.role in settings.settings.admin_roles:
return db.update_group_author(conn, groupname, new_author)
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return db.update_group_author(conn, groupname, new_author)
else:
raise HTTPException(
@ -151,8 +159,6 @@ async def update_invite_code(
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
group = get_group_by_name(conn, groupname)
invite_code = "".join(secrets.choice(JOIN_CODE_SYMBOLS) for _ in range(startup_settings.join_code_length))
while db.check_invite_code(conn, invite_code):
invite_code = "".join(secrets.choice(JOIN_CODE_SYMBOLS) for _ in range(startup_settings.join_code_length))
@ -162,7 +168,7 @@ async def update_invite_code(
"result": db.update_group_invite_code(conn, groupname, invite_code),
"invite code": invite_code
}
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return {
"result": db.update_group_invite_code(conn, groupname, invite_code),
"invite code": invite_code
@ -181,10 +187,9 @@ async def update_allow_skips(
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
group = get_group_by_name(conn, groupname)
if current_user.role in settings.settings.admin_roles:
return db.update_group_allow_skips(conn, groupname, allow_skips)
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return db.update_group_allow_skips(conn, groupname, allow_skips)
else:
raise HTTPException(
@ -200,10 +205,9 @@ async def update_feed_interval(
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
group = get_group_by_name(conn, groupname)
if current_user.role in settings.settings.admin_roles:
return db.update_group_feed_interval(conn, groupname, feed_interval)
if current_user.username == group.author:
if db.check_group_author(conn, groupname, current_user.username):
return db.update_group_feed_interval(conn, groupname, feed_interval)
else:
raise HTTPException(

View File

@ -6,8 +6,12 @@ from psycopg2._psycopg import connection
import db.memberships as db
import settings.settings as settings
from api.models import User
from api.utils import get_current_user, get_group_by_name
from db.groups import check_group_existence, get_groupname_by_invite_code
from api.utils import get_current_user
from db.groups import (
check_group_author,
check_group_existence,
get_groupname_by_invite_code,
)
from db.internal import get_db_connection
from db.users import check_user_existence
@ -107,8 +111,7 @@ async def delete_membership(
if current_user.role in settings.settings.admin_roles:
return db.delete_membership(conn, username, groupname)
group = get_group_by_name(conn, groupname)
if current_user.username == group.author:
if check_group_author(conn, groupname, current_user.username):
return db.delete_membership(conn, username, groupname)
else:
raise HTTPException(

View File

@ -2,6 +2,8 @@ from datetime import datetime
from pydantic import BaseModel
from api.string_tools import decode_str
class Token(BaseModel):
access_token: str
@ -18,14 +20,13 @@ class User(BaseModel):
self.password = params["password"]
self.role = params["role"]
self.disabled = params["disabled"]
self.groups_ids = params["groups_ids"]
self.last_seen_at = params["last_seen_at"]
self.created_at = params["created_at"]
return self
username: str = ""
password: str = ""
role: str = "user"
disabled: bool = False
groups_ids: list[str] | None = None
last_seen_at: datetime | None = None
created_at: datetime | None = None
@ -39,6 +40,7 @@ class Group(BaseModel):
self.feed_interval_minutes = params["feed_interval_minutes"]
self.last_feed_id = params["last_feed_id"]
self.created_at = params["created_at"]
return self
groupname: str = ""
author: str = ""
invite_code: str = ""
@ -53,6 +55,7 @@ class Membership(BaseModel):
self.groupname = params["groupname"]
self.username = params["username"]
self.joined_at = params["joined_at"]
return self
groupname: str = ""
username: str = ""
joined_at: datetime | None = None
@ -66,9 +69,59 @@ class Picture(BaseModel):
self.url = params["url"]
self.metadata = params["metadata"]
self.created_at = params["created_at"]
return self
id: int = -1
source: str = ""
external_id: str = ""
url: str = ""
metadata: dict | None = None
metadata: dict = {}
created_at: datetime | None = None
class Swipe(BaseModel):
def fill(self, params):
self.username = params["username"]
self.feed_id = params["feed_id"]
self.picture_id = params["picture_id"]
self.value = params["value"]
self.created_at = params["created_at"]
return self
username: str = ""
feed_id: int = -1
picture_id: int = -1
value: int = 0
created_at: datetime | None = None
class Feed(BaseModel):
def fill(self, params):
self.id = params["id"]
self.groupname = params["groupname"]
self.image_ids = params["image_ids"]
self.created_at = params["created_at"]
return self
id: int = -1
groupname: str = ""
image_ids: list[int] = []
created_at: datetime | None = None
class Account(BaseModel):
def fill(self, params):
self.id = params["id"]
self.groupname = params["groupname"]
self.author = params["author"]
self.platform = params["platform"]
self.login = params["login"]
self.password = decode_str(params["password"])
self.metadata = params["metadata"]
self.created_at = params["created_at"]
return self
id: int = -1
groupname: str = ""
author: str = ""
platform: str = ""
login: str = ""
password: str = ""
metadata: dict = {}
created_at: datetime | None = None

View File

@ -12,39 +12,19 @@ from db.internal import get_db_connection
pictures_router = APIRouter(prefix="/api/pictures", tags=["pictures"])
# maybe to delete
@pictures_router.post("/picture/url")
async def read_picture_by_url(
groupname: str,
@pictures_router.post("/picture")
async def read_picture(
id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
picture = Picture()
picture_data = db.get_picture_by_url(conn, groupname)
picture_data = db.get_picture(conn, id)
if picture_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such picture",
)
picture.fill(picture_data)
return picture
@pictures_router.post("/picture/id")
async def read_picture_by_id(
groupname: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
picture = Picture()
picture_data = db.get_picture_by_id(conn, groupname)
if picture_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such picture",
)
picture.fill(picture_data)
return picture
return Picture().fill(picture_data)
@pictures_router.post("/add")
@ -62,40 +42,19 @@ async def add_picture(
detail="Not allowed",
)
# if db.check_picture_existence(conn, groupname):
# raise HTTPException(
# status_code=status.HTTP_409_CONFLICT,
# detail="Picture already exists",
# )
return {
"id": db.create_picture(conn, source, external_id, url, metadata)
}
# maybe to delete
@pictures_router.post("/delete/url")
async def delete_picture_by_url(
picture_url: str,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if current_user.role in settings.settings.admin_roles:
return db.delete_picture_by_url(conn, picture_url)
else:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
@pictures_router.post("/delete/id")
async def delete_picture_by_id(
@pictures_router.post("/delete")
async def delete_picture(
picture_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if current_user.role in settings.settings.admin_roles:
return db.delete_picture_by_id(conn, picture_id)
return db.delete_picture(conn, picture_id)
else:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,

22
src/api/string_tools.py Normal file
View File

@ -0,0 +1,22 @@
import settings.startup_settings as startup_settings
def encode_str(string) -> str:
key = startup_settings.secret_key
encoded_chars = []
for i in range(len(string)):
key_c = key[i % len(key)]
encoded_c = chr(ord(string[i]) + ord(key_c) % 256)
encoded_chars.append(encoded_c)
encoded_string = ''.join(encoded_chars)
return encoded_string
def decode_str(encoded_string) -> str:
key = startup_settings.secret_key
encoded_chars = []
for i in range(len(encoded_string)):
key_c = key[i % len(key)]
encoded_c = chr((ord(encoded_string[i]) - ord(key_c) + 256) % 256)
encoded_chars.append(encoded_c)
decoded_string = ''.join(encoded_chars)
return decoded_string

144
src/api/swipes.py Normal file
View File

@ -0,0 +1,144 @@
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, status
from psycopg2._psycopg import connection
import db.swipes as db
import settings.settings as settings
from api.models import Group, Swipe, User
from api.utils import get_current_user
from db.feeds import get_groupname_by_feed_id
from db.groups import get_group
from db.internal import get_db_connection
from db.memberships import check_membership_exists
swipes_router = APIRouter(prefix="/api/swipes", tags=["swipes"])
# Maybe endpoints should be remade
# to return id and then work with swipe id
@swipes_router.post("/swipe")
async def read_swipe(
username: str,
feed_id: int,
picture_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
groupname = get_groupname_by_feed_id(conn, feed_id)
if groupname is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed or feed is not linked to group",
)
if not check_membership_exists(conn, current_user.username, groupname) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
swipe_data = db.get_swipe(conn, username, feed_id, picture_id)
if swipe_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such swipe",
)
return Swipe().fill(swipe_data)
@swipes_router.post("/swipe/picture_id")
async def read_swipes_by_picture_id(
feed_id: int,
picture_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
groupname = get_groupname_by_feed_id(conn, feed_id)
if groupname is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed or feed is not linked to group",
)
if not check_membership_exists(conn, current_user.username, groupname) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return db.get_swipes_by_picture_id(conn, picture_id, feed_id)
@swipes_router.post("/swipe/user")
async def read_user_swipes(
username: str,
feed_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
groupname = get_groupname_by_feed_id(conn, feed_id)
if groupname is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed or feed is not linked to group",
)
if not check_membership_exists(conn, current_user.username, groupname) and current_user.role not in settings.settings.admin_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
return db.get_swipes_by_user(conn, username, feed_id)
@swipes_router.post("/add")
async def add_swipe(
feed_id: int,
picture_id: int,
value: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
groupname = get_groupname_by_feed_id(conn, feed_id)
if groupname is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed or feed is not linked to group",
)
group_data = get_group(conn, groupname)
if group_data is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such feed or feed is not linked to group",
)
group = Group().fill(group_data)
# Check for trying to skip in
# a group with skips disabled
if value == 0 and group.allow_skips is False:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Skips are disallowed in the group",
)
result = db.create_swipe(conn, current_user.username, feed_id, picture_id, value)
if result:
# TODO: call function to like picture on the platform
pass
return result
@swipes_router.post("/delete")
async def delete_swipe(
username: str,
feed_id: int,
picture_id: int,
conn: Annotated[connection, Depends(get_db_connection)],
current_user: Annotated[User, Depends(get_current_user)]
):
if current_user.role in settings.settings.admin_roles:
return db.delete_swipe(conn, username, feed_id, picture_id)
else:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)

View File

@ -27,14 +27,13 @@ async def read_users_any(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not allowed",
)
user = User()
user_data = db.get_user(conn, username)
if user_data is None:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such user",
)
user.fill(user_data)
user = User().fill(user_data)
return user

View File

@ -6,8 +6,6 @@ import jwt
from fastapi import Depends, HTTPException, status
from fastapi.security import OAuth2PasswordBearer
from jwt.exceptions import InvalidTokenError
# from passlib.context import CryptContext
from psycopg2._psycopg import connection
import db.groups
@ -16,8 +14,6 @@ import settings.startup_settings as startup_settings
from api.models import Group, TokenData, User
from db.internal import get_db_connection
# pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
@ -81,12 +77,11 @@ async def get_current_user(
except InvalidTokenError:
raise credentials_exception
user = User()
user_data = db.users.get_user(conn, token_data.username)
if user_data is None:
raise credentials_exception
user.fill(user_data)
user = User().fill(user_data)
if user.disabled:
raise HTTPException(
@ -96,6 +91,7 @@ async def get_current_user(
return user
def get_group_by_name(
conn: connection,
groupname: str
@ -104,10 +100,8 @@ def get_group_by_name(
status_code=status.HTTP_404_NOT_FOUND,
detail="No such group"
)
group = Group()
group_data = db.groups.get_group(conn, groupname)
if group_data is None:
raise group_exception
group.fill(group_data)
return group
return Group().fill(group_data)

View File

@ -3,16 +3,20 @@ import sys
from fastapi import FastAPI
from loguru import logger
from api.accounts import accounts_router
from api.anon import anon_router
from api.auth import auth_router
from api.feeds import feeds_router
from api.general import general_router
from api.groups import groups_router
from api.memberships import memberships_router
from api.pictures import pictures_router
from api.swipes import swipes_router
from api.users import users_router
from db.internal import connect_db, disconnect_db
from settings import startup_settings
from settings.settings import settings_down, settings_up
from settings.startup_settings import setup_settings
docs_url = None
if startup_settings.swagger_enabled:
@ -30,11 +34,12 @@ def create_app():
{
"sink": sys.stdout,
"level": startup_settings.log_level,
"format": "<level>{level}: {message}</level>",
"format": "<level>{level}</level>: {message}",
}
]
)
app.add_event_handler("startup", setup_settings)
app.add_event_handler("startup", connect_db)
app.add_event_handler("startup", settings_up)
@ -44,6 +49,9 @@ def create_app():
app.include_router(users_router)
app.include_router(groups_router)
app.include_router(memberships_router)
app.include_router(accounts_router)
app.include_router(feeds_router)
app.include_router(swipes_router)
app.include_router(pictures_router)
app.add_event_handler("shutdown", disconnect_db)

154
src/db/accounts.py Normal file
View File

@ -0,0 +1,154 @@
import json
import psycopg2.extras
from psycopg2._psycopg import connection
from api.models import Account
# account create and delete
def create_account(
conn: connection,
groupname: str,
author: str,
platform: str,
login: str,
password: str,
metadata: dict
):
with conn.cursor() as cur:
cur.execute(
"""
insert into picrinth.accounts
(groupname, author, platform, login,
password, metadata, created_at)
values (%s, %s, %s, %s, now())
returning id
""",
(groupname, author, platform, login, password, json.dumps(metadata)),
)
result = cur.fetchone()
conn.commit()
if result is None:
return None
return result[0]
def delete_account(
conn: connection,
account_id: str
):
with conn.cursor() as cur:
cur.execute(
"""
delete from picrinth.accounts
where account_id = %s
""",
(account_id),
)
conn.commit()
return cur.rowcount > 0
# account checks
def check_account_existence(
conn: connection,
groupname: str,
platform: str
):
with conn.cursor() as cur:
cur.execute(
"""
select exists(
select 1
from picrinth.accounts
where groupname = %s and platform = %s
);
""",
(groupname, platform),
)
return cur.fetchone()[0] # type: ignore
# account update
def update_account(
conn: connection,
groupname: str,
author: str,
platform: str,
login: str,
password: str,
metadata: dict
):
with conn.cursor() as cur:
cur.execute(
"""
update picrinth.accounts
SET author = %s,
login = %s,
password = %s,
metadata = %s
where groupname = %s and platform = %s
""",
(author, login, password, json.dumps(metadata), groupname, platform),
)
conn.commit()
return cur.rowcount > 0
def update_account_metadata(
conn: connection,
groupname: str,
platform: str,
metadata: dict
):
with conn.cursor() as cur:
cur.execute(
"""
update picrinth.accounts
SET metadata = %s
where groupname = %s and platform = %s
""",
(json.dumps(metadata), groupname, platform),
)
conn.commit()
return cur.rowcount > 0
# account receiving
def get_accounts_by_group(
conn: connection,
groupname: str
) -> list[Account]:
with conn.cursor() as cur:
cur.execute(
"""
select *
from picrinth.accounts
where groupname = %s
""",
(groupname,),
)
return [Account().fill(account_data) for (account_data,) in cur.fetchall()]
def get_accounts_by_group_platform(
conn: connection,
groupname: str,
platform: str
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""
select groupname, author,
platform, login, password,
metadata, created_at
from picrinth.accounts
where groupname = %s and platform = %s
""",
(groupname, platform),
)
return cur.fetchone()

83
src/db/feeds.py Normal file
View File

@ -0,0 +1,83 @@
import psycopg2.extras
from psycopg2._psycopg import connection
# feed create and delete
def create_feed(
conn: connection,
groupname: str,
image_ids: list[int]
):
with conn.cursor() as cur:
cur.execute(
"""
insert into picrinth.feeds
(groupname, image_ids, created_at)
values (%s, %s, now())
returning id
""",
(groupname, image_ids),
)
result = cur.fetchone()
conn.commit()
if result is None:
return None
return result[0]
def delete_feed(
conn: connection,
feed_id: int
):
with conn.cursor() as cur:
cur.execute(
"""
delete from picrinth.feeds
where id = %s
""",
(feed_id,),
)
conn.commit()
return cur.rowcount > 0
# feed receiving
def get_feed(
conn: connection,
feed_id: int
):
with conn.cursor() as cur:
cur.execute(
"""
select groupname
from picrinth.feeds
where id = %s
""",
(feed_id,),
)
result = cur.fetchone()
if result is None:
return None
return cur.fetchone()[0] # type: ignore
# additional
def get_groupname_by_feed_id(
conn: connection,
feed_id: int
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""
select groupname
from picrinth.feeds
where id = %s
""",
(feed_id,),
)
result = cur.fetchone()
if result is None:
return None
return cur.fetchone()[0] # type: ignore

View File

@ -79,6 +79,25 @@ def check_invite_code(
return cur.fetchone()[0] # type: ignore
def check_group_author(
conn: connection,
groupname: str,
author: str
):
with conn.cursor() as cur:
cur.execute(
"""
select exists(
select 1
from picrinth.groups
where groupname = %s and author = %s
);
""",
(groupname, author),
)
return cur.fetchone()[0] # type: ignore
# group updates
def update_group_groupname(
@ -243,6 +262,7 @@ def get_groupname_by_invite_code(
return None
return result[0] # type: ignore
def get_group_invite_code(
conn: connection,
groupname: str
@ -260,3 +280,21 @@ def get_group_invite_code(
if result is None:
return None
return result[0] # type: ignore
def get_group_last_feed_id(
conn: connection,
groupname: str
):
with conn.cursor() as cur:
cur.execute(
"""
select last_feed_id
from picrinth.groups
where groupname = %s
""",
(groupname,),
)
result = cur.fetchone()
if result is None:
return None
return result[0] # type: ignore

View File

@ -1,4 +1,3 @@
import psycopg2.extras
from psycopg2._psycopg import connection
# membership create and delete
@ -65,7 +64,7 @@ def get_memberships_by_username(
conn: connection,
username: str
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
with conn.cursor() as cur:
cur.execute(
"""
select *
@ -81,7 +80,7 @@ def get_memberships_by_groupname(
conn: connection,
groupname: str
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
with conn.cursor() as cur:
cur.execute(
"""
select *

View File

@ -29,22 +29,7 @@ def create_picture(
return result[0]
def delete_picture_by_url(
conn: connection,
url: str
):
with conn.cursor() as cur:
cur.execute(
"""
delete from picrinth.pictures
where url = %s
""",
(url,),
)
conn.commit()
return cur.rowcount > 0
def delete_picture_by_id(
def delete_picture(
conn: connection,
id: int
):
@ -62,26 +47,9 @@ def delete_picture_by_id(
# picture receiving
def get_picture_by_url(
def get_picture(
conn: connection,
url: str
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""
select id, source,
external_id, url,
metadata, created_at
from picrinth.pictures
where url = %s
""",
(url,),
)
return cur.fetchone()
def get_picture_by_id(
conn: connection,
id: str
id: int
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
@ -95,3 +63,16 @@ def get_picture_by_id(
(id,),
)
return cur.fetchone()
def get_all_pictures_ids(
conn: connection,
):
with conn.cursor() as cur:
cur.execute(
"""
select id
from picrinth.pictures
""",
)
return [element for (element,) in cur.fetchall()]

100
src/db/swipes.py Normal file
View File

@ -0,0 +1,100 @@
import psycopg2.extras
from psycopg2._psycopg import connection
# swipe create and delete
def create_swipe(
conn: connection,
username: str,
feed_id: int,
picture_id: int,
value: int
):
with conn.cursor() as cur:
cur.execute(
"""
insert into picrinth.swipes
(username, feed_id, picture_id, value, created_at)
values (%s, %s, %s, %s, now())
returning id
""",
(username, feed_id, picture_id, value),
)
result = cur.fetchone()
conn.commit()
if result is None:
return None
return result[0]
def delete_swipe(
conn: connection,
username: str,
feed_id: int,
picture_id: int
):
with conn.cursor() as cur:
cur.execute(
"""
delete from picrinth.swipes
where username = %s and picture_id = %s and feed_id = %s
""",
(username, picture_id, feed_id),
)
conn.commit()
return cur.rowcount > 0
# swipe receiving
def get_swipe(
conn: connection,
username: str,
feed_id: int,
picture_id: int
):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""
select username,
feed_id, picture_id,
value, created_at
from picrinth.swipes
where username = %s and feed_id = %s and picture_id = %s
""",
(username, feed_id, picture_id),
)
return cur.fetchone()
def get_swipes_by_picture_id(
conn: connection,
feed_id: int,
picture_id: int
):
with conn.cursor() as cur:
cur.execute(
"""
select *
from picrinth.swipes
where feed_id = %s and picture_id = %s
""",
(feed_id, picture_id),
)
return cur.fetchone()
def get_swipes_by_user(
conn: connection,
username: str,
feed_id: int
):
with conn.cursor() as cur:
cur.execute(
"""
select *
from picrinth.swipes
where username = %s and feed_id = %s
""",
(username, feed_id),
)
return cur.fetchone()

View File

@ -176,8 +176,7 @@ def get_user(
cur.execute(
"""
select username, password, role,
disabled, groups_ids,
last_seen_at, created_at
disabled, last_seen_at, created_at
from picrinth.users
where username = %s
""",

10
src/scraper/gelbooru.py Normal file
View File

@ -0,0 +1,10 @@
from psycopg2._psycopg import connection
import db.pictures as db
from api.models import Account, Picture
def gelbooru(conn: connection, account: Account):
picture = Picture(external_id = "3", url = "", metadata = {})
picture_id = db.create_picture(conn, "gelbooru", picture.external_id, picture.url, picture.metadata)
return picture_id

10
src/scraper/pinterest.py Normal file
View File

@ -0,0 +1,10 @@
from psycopg2._psycopg import connection
import db.pictures as db
from api.models import Account, Picture
def pinterest(conn: connection, account: Account):
picture = Picture(external_id = "1", url = "", metadata = {})
picture_id = db.create_picture(conn, "pinterest", picture.external_id, picture.url, picture.metadata)
return picture_id

132
src/scraper/pixiv.py Normal file
View File

@ -0,0 +1,132 @@
from fastapi import HTTPException, status
from gppt import GetPixivToken
from loguru import logger
from pixivpy3 import AppPixivAPI
from psycopg2._psycopg import connection
import db.pictures as db
from api.models import Account, Picture
from db.accounts import update_account_metadata
# Wrapper functions
def pixiv(conn: connection, account: Account) -> list[int]:
# Getting refresh token
refresh_token = account.metadata.get('refresh_token', '')
if not refresh_token:
try:
refresh_token = get_refresh_token(account.login, account.password)
account.metadata['refresh_token'] = refresh_token
update_account_metadata(conn, account.groupname, account.platform, account.metadata)
except Exception as e:
# TODO: review ruff "do not use bare `except`"
logger.debug(f"Pixiv refresh token missing and creation failed: {e}")
raise HTTPException(
status_code=status.HTTP_407_PROXY_AUTHENTICATION_REQUIRED,
detail="Pixiv refresh token missing and creation failed"
)
# Logging into pixiv
try:
api = auth(refresh_token)
except Exception as e:
# TODO: review ruff "do not use bare `except`"
logger.debug(f"Pixiv refresh token invalid and recreation failed: {e}")
raise HTTPException(
status_code=status.HTTP_407_PROXY_AUTHENTICATION_REQUIRED,
detail="Pixiv refresh token invalid and recreation failed"
)
# Getting pixiv account recommendations
pictures = []
try:
pictures = get_recommended(api, 20)
except Exception as e:
logger.error(f"Failed to get recommendations from pixiv: {e}")
if not pictures:
logger.error("Failed to generate feed from pixiv")
return []
# Saving recommendations as Pictures to DB
pictures_ids = []
for picture in pictures:
pictures_ids.append(db.create_picture(conn, "pixiv", picture.external_id, picture.url, picture.metadata))
return pictures_ids
def like_picture(conn: connection, account: Account, picture_id: int):
# Getting refresh token
refresh_token = account.metadata.get('refresh_token', '')
if not refresh_token:
try:
refresh_token = get_refresh_token(account.login, account.password)
account.metadata['refresh_token'] = refresh_token
update_account_metadata(conn, account.groupname, account.platform, account.metadata)
except Exception as e:
# TODO: review ruff "do not use bare `except`"
logger.debug(f"Pixiv refresh token missing and creation failed: {e}")
raise HTTPException(
status_code=status.HTTP_407_PROXY_AUTHENTICATION_REQUIRED,
detail="Pixiv refresh token missing and creation failed"
)
# Logging into pixiv
try:
api = auth(refresh_token)
except Exception as e:
# TODO: review ruff "do not use bare `except`"
logger.debug(f"Pixiv refresh token invalid and recreation failed: {e}")
raise HTTPException(
status_code=status.HTTP_407_PROXY_AUTHENTICATION_REQUIRED,
detail="Pixiv refresh token invalid and recreation failed"
)
# Liking
return like(api, picture_id)
# Auth
def get_refresh_token(login: str, password: str) -> str:
g = GetPixivToken(headless=True)
login_response = g.login(username=login, password=password)
refresh_token = login_response.get("refresh_token", '')
if not refresh_token:
raise Exception
return refresh_token
def auth(refresh_token: str) -> AppPixivAPI:
api = AppPixivAPI()
api.auth(refresh_token)
print("Login successful") # TODO: delete
return api
def refresh_refresh_token(refresh_token: str) -> str:
g = GetPixivToken(headless=True)
# TODO: save new refresh_token to DB + account + add auto updating as coroutine
response = g.refresh(refresh_token)
print(response) # TODO: delete
return response.get("refresh_token", '')
# Main functions
def get_recommended(api: AppPixivAPI, recommendations_number: int = 20) -> list[Picture]:
result = api.illust_recommended()
# TODO: make recommendations number useful + add as var to settings/group settings
# illusts = result.illusts[:recommendations_number]
illusts = result.illusts
pictures = []
for illust in illusts:
if illust.type == "illust":
picture = Picture(external_id=illust.id, source="pixiv", url=illust.image_urls.large, metadata={})
pictures.append(picture)
return pictures
def like(api: AppPixivAPI, illust_id: int):
api.illust_bookmark_add(illust_id)
print(f"Picture {illust_id} liked!") # TODO: delete

53
src/scraper/utils.py Normal file
View File

@ -0,0 +1,53 @@
from loguru import logger
from psycopg2._psycopg import connection
import db.pictures as db
from api.models import Account, Picture
from scraper.gelbooru import gelbooru
from scraper.pinterest import pinterest
from scraper.pixiv import pixiv
from settings import startup_settings
def generate_feed(
conn: connection,
accounts: list[Account]
) -> list:
feed = []
for account in accounts:
if account.platform not in startup_settings.platforms_enabled:
raise Exception
match account.platform:
case "pinterest":
temp_feed = pinterest(conn, account)
feed.append(temp_feed)
case "pixiv":
temp_feed = pixiv(conn, account)
feed.append(temp_feed)
case "gelbooru":
temp_feed = gelbooru(conn, account)
feed.append(temp_feed)
case _:
logger.warning(f"Platform for feed generation is not supported: {account.platform}")
# TODO: remove mock results
feed = db.get_all_pictures_ids(conn)
return feed
# TODO: Implement process_swipe
def process_swipe():
return
def get_picture(
conn: connection,
picture_id: int
) -> int:
picture_data = db.get_picture(conn, picture_id)
if picture_id is None:
return -1
return Picture().fill(picture_data).id

View File

@ -1 +1,20 @@
JOIN_CODE_SYMBOLS = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" # No O + 0, I + 1
# No O + 0, I + 1. All in upper case
JOIN_CODE_SYMBOLS = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"
# All platforms should be written in lower case
SUPPORTED_PLATFORMS = ["pinterest", "pixiv", "gelbooru"]
# Info for settings update endpoint
API_EDITABLE_SETTINGS_LIST = """
admin_roles ["admin"]
allow_create_admins_by_admins True
allow_create_admins True
allow_create_users True
allow_create_groups True
allow_create_pictures True
"""

View File

@ -12,7 +12,7 @@ class Settings(BaseModel):
self.allow_create_admins = params["allow_create_admins"]
self.allow_create_users = params["allow_create_users"]
self.allow_create_groups = params["allow_create_groups"]
self.allow_create_pictures = params["allow_create_groups"]
self.allow_create_pictures = params["allow_create_pictures"]
admin_roles: list[str] = ["admin"]
allow_create_admins_by_admins: bool = True
allow_create_admins: bool = True
@ -61,9 +61,7 @@ def settings_down():
def reset_settings():
global settings, json_path, json_settings_name
logger.info("Resetting settings")
print(settings)
settings = Settings()
print(settings)
with open(json_path + json_settings_name, "w") as f:
json.dump(settings.model_dump_json(), f, ensure_ascii = False, indent=4)
logger.info("Wrote settings to the JSON")

View File

@ -1,24 +1,36 @@
from decouple import config
from decouple import Csv, config
from loguru import logger
def str_to_bool(string: str) -> bool:
if string.lower() == "true":
return True
return False
from .consts import SUPPORTED_PLATFORMS
# database
db_host = str(config("db_host", default="127.0.0.1"))
db_port = int(config("db_port", default=5432))
db_name = str(config("db_name", default="postgres"))
db_user = str(config("db_user", default="postgres"))
db_password = str(config("db_password", default="postgres"))
db_host: str = config("db_host", default="127.0.0.1", cast=str) # type: ignore
db_port: int = config("db_port", default=5432, cast=int) # type: ignore
db_name: str = config("db_name", default="postgres", cast=str) # type: ignore
db_user: str = config("db_user", default="postgres", cast=str) # type: ignore
db_password: str = config("db_password", default="postgres", cast=str) # type: ignore
# auth
secret_key = str(config("secret_key"))
algorithm = str(config("algorithm", "HS256"))
access_token_expiration_time = int(config("access_token_expiration_time", default=10080))
secret_key: str = config("secret_key", cast=str) # type: ignore
algorithm: str = config("algorithm", "HS256", cast=str) # type: ignore
access_token_expiration_time: int = config("access_token_expiration_time", default=10080, cast=int) # type: ignore
# other settings
swagger_enabled = str_to_bool(str(config("swagger_enabled", "false")))
log_level = str(config("log_level", default="INFO"))
join_code_length = int(config("join_code_length", default=8))
join_code_length: int = config("join_code_length", default=8, cast=int) # type: ignore
platforms_enabled: list[str] = config("platforms_enabled", default=SUPPORTED_PLATFORMS, cast=Csv(str)) # type: ignore
# dev
swagger_enabled: bool = config("swagger_enabled", "false", cast=bool) # type: ignore
log_level: str = config("log_level", default="INFO", cast=str) # type: ignore
def setup_settings():
global platforms_enabled
platforms_supported_and_enabled = []
for platform in platforms_enabled:
if platform.lower() in SUPPORTED_PLATFORMS:
platforms_supported_and_enabled.append(platform.lower())
else:
logger.warning(f"Platform {platform} is not supported by design. It will not be used.")
logger.info(f"Supported platforms list: {platforms_supported_and_enabled}")
platforms_enabled = platforms_supported_and_enabled

View File

@ -37,7 +37,7 @@ create table picrinth.pictures (
id serial not null,
source text not null,
external_id text not null,
url text not null,
"url" text not null,
metadata jsonb null,
created_at timestamp with time zone default now(),
constraint pictures_pkey primary key (id),
@ -59,11 +59,24 @@ create table picrinth.swipes (
username text not null,
feed_id integer not null,
picture_id integer not null,
swipe_value smallint not null,
swiped_at timestamp with time zone default now(),
"value" smallint not null,
created_at timestamp with time zone default now(),
primary key (id),
foreign key (username) references picrinth.users (username) on delete cascade on update cascade,
foreign key (feed_id) references picrinth.feeds (id) on delete cascade on update cascade,
foreign key (picture_id) references picrinth.pictures (id) on delete cascade on update cascade,
constraint swipes_unique unique (username, feed_id, picture_id)
);
create table picrinth.accounts (
groupname text not null,
author text null,
platform text not null,
"login" text not null,
"password" text not null,
metadata jsonb null,
created_at timestamp with time zone default now(),
foreign key (groupname) references picrinth.groups (groupname) on delete cascade on update cascade,
foreign key (author) references picrinth.groups (author) on delete cascade on update cascade,
constraint unique_account_for_group_per_platform unique (groupname, platform)
);