fix: extension + file permissions

botlib/curl.py

@@ -48,14 +48,40 @@ class Curl:
                               md5(url.encode()).hexdigest())
 
     @staticmethod
-    def open(url: str, *, headers: Optional[Dict[str, str]] = None) \
+    def _cached_is_recent(fname: str, *, maxAge: int) -> bool:
-            -> Optional[HTTPResponse]:
+        fname = os.path.join(Curl.CACHE_DIR, fname)
+        return os.path.isfile(fname) and FileTime.get(fname) < maxAge
+
+    @staticmethod
+    def _cached_read(
+        conn: Optional[HTTPResponse], fname_data: str, fname_head: str
+    ) -> Optional[TextIO]:
+        fname_data = os.path.join(Curl.CACHE_DIR, fname_data)
+        if conn:
+            os.makedirs(Curl.CACHE_DIR, exist_ok=True)
+            with open(os.path.join(Curl.CACHE_DIR, fname_head), 'w') as fp:
+                fp.write(str(conn.info()).strip())
+            with open(fname_data, 'wb') as fpb:
+                while True:
+                    data = conn.read(8192)  # 1024 Bytes
+                    if not data:
+                        break
+                    fpb.write(data)
+        return open(fname_data) if os.path.isfile(fname_data) else None
+
+    @staticmethod
+    def open(
+        url: str,
+        *,
+        post: Optional[bytes] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Optional[HTTPResponse]:
         ''' Open a network connection, returl urlopen() result or None. '''
         try:
             head = {'User-Agent': 'Mozilla/5.0'}
             if headers:
                 head.update(headers)
-            return urlopen(Request(url, headers=head))
+            return urlopen(Request(url, data=post, headers=head))
         except Exception as e:
             if isinstance(e, HTTPError) and e.getcode() == 304:
                 # print('Not-Modified: {}'.format(url), file=stderr)
@@ -64,31 +90,48 @@ class Curl:
             return None
 
     @staticmethod
-    def get(url: str, *, cache_only: bool = False) -> Optional[TextIO]:
+    def get(
+        url: str,
+        *,
+        cache_only: bool = False,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Optional[TextIO]:
         '''
         Returns an already open file pointer.
         You are responsible for closing the file.
         NOTE: `HTML2List.parse` and `Feed2List.parse` will close it for you.
         '''
-        fname = '{}/curl-{}.data'.format(Curl.CACHE_DIR, Curl.url_hash(url))
+        fname = 'curl-{}.data'.format(Curl.url_hash(url))
-        fname_head = fname[:-5] + '.head'
         # If file was created less than 45 sec ago, reuse cached value
-        if cache_only or (os.path.isfile(fname) and FileTime.get(fname) < 45):
+        if cache_only or Curl._cached_is_recent(fname, maxAge=45):
-            return open(fname)
+            return Curl._cached_read(None, fname, '')
 
-        os.makedirs(Curl.CACHE_DIR, exist_ok=True)
+        fname_head = fname[:-5] + '.head'
-        conn = Curl.open(url, headers=_read_modified_header(fname_head))
+        head = _read_modified_header(fname_head)
-        if conn:
+        if headers:
-            with open(fname_head, 'w') as fp:
+            head.update(headers)
-                fp.write(str(conn.info()).strip())
+        conn = Curl.open(url, headers=head)
-            with open(fname, 'wb') as fpb:
+        return Curl._cached_read(conn, fname, fname_head)
-                while True:
-                    data = conn.read(8192)  # 1024 Bytes
-                    if not data:
-                        break
-                    fpb.write(data)
 
-        return open(fname) if os.path.isfile(fname) else None
+    @staticmethod
+    def post(
+        url: str,
+        data: bytes,
+        *,
+        cache_only: bool = False,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Optional[TextIO]:
+        '''
+        Perform POST operation.
+        Returns an already open file pointer.
+        You are responsible for closing the file.
+        '''
+        fname = 'curl-{}.post.data'.format(Curl.url_hash(url))
+        if cache_only:
+            return Curl._cached_read(None, fname, '')
+
+        conn = Curl.open(url, post=data, headers=headers)
+        return Curl._cached_read(conn, fname, fname[:-5] + '.head')
 
     @staticmethod
     def json(url: str, fallback: Any = None, *, cache_only: bool = False) \
@@ -141,7 +184,9 @@ class Curl:
             if not parts:
                 raise URLError('URL not valid: "{}"'.format(url_str))
 
-            ext = parts.path.split('.')[-1] or 'unknown'
+            ext = parts.path.split('.')[-1]
+            if not ext or len(ext) > 4:
+                ext = 'unknown'
             file_path = os.path.join(dest_dir, fname + '.' + ext)
             if override or not os.path.isfile(file_path):
                 url = parts.geturl()

botlib/helper.py

@@ -2,6 +2,7 @@
 import re
 import os  # utime, getmtime
 import time  # mktime, time
+import traceback  # format_exc
 from sys import stderr
 from html import unescape
 from datetime import datetime
@@ -11,15 +12,32 @@ from typing import Optional, Callable, Union
 
 
 class Log:
+    FILE = 'error.log'
+    LEVEL = 0  # -1: disabled, 0: error, 1: warn, 2: info, 4: debug
+
     @staticmethod
-    def error(e: str) -> None:
+    def _log_if(level: int, msg: str) -> None:
+        ''' Log to file if LOG_LEVEL >= level. '''
+        if Log.LEVEL >= level:
+            with open(Log.FILE, 'a') as fp:
+                fp.write(msg + '\n')
+
+    @staticmethod
+    def error(e: Union[str, Exception]) -> None:
         ''' Log error message (incl. current timestamp) '''
-        print('{} [ERROR] {}'.format(datetime.now(), e), file=stderr)
+        msg = '{} [ERROR] {}'.format(
+            datetime.now(), e if isinstance(e, str) else repr(e))
+        print(msg, file=stderr)
+        Log._log_if(0, msg)
+        if isinstance(e, Exception):
+            Log._log_if(0, traceback.format_exc())
 
     @staticmethod
     def info(m: str) -> None:
         ''' Log info message (incl. current timestamp) '''
-        print('{} {}'.format(datetime.now(), m))
+        msg = '{} {}'.format(datetime.now(), m)
+        print(msg)
+        Log._log_if(2, msg)
 
 
 class FileTime:

botlib/tgclient.py

@@ -42,7 +42,7 @@ class TGClient(telebot.TeleBot):
                         self.onKillCallback()
                     return
                 except Exception as e:
-                    Log.error(repr(e))
+                    Log.error(e)
                     Log.info('Auto-restart in 15 sec ...')
                     sleep(15)
                     _fn()
@@ -91,7 +91,7 @@ class TGClient(telebot.TeleBot):
         try:
             return self.send_message(chat_id, msg, **kwargs)
         except Exception as e:
-            Log.error(repr(e))
+            Log.error(e)
             sleep(45)
             return None