fix: unable to find byted acrawler

README.md

@@ -4,11 +4,11 @@ This is an unofficial api wrapper for TikTok.com in python. With this api you ar
 
 [![DOI](https://zenodo.org/badge/188710490.svg)](https://zenodo.org/badge/latestdoi/188710490) [![LinkedIn](https://img.shields.io/badge/LinkedIn-0077B5?style=for-the-badge&logo=linkedin&logoColor=white&style=flat-square)](https://www.linkedin.com/in/davidteather/) [![Sponsor Me](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub)](https://github.com/sponsors/davidteather) [![GitHub release (latest by date)](https://img.shields.io/github/v/release/davidteather/TikTok-Api)](https://github.com/davidteather/TikTok-Api/releases) [![GitHub](https://img.shields.io/github/license/davidteather/TikTok-Api)](https://github.com/davidteather/TikTok-Api/blob/main/LICENSE) [![Downloads](https://pepy.tech/badge/tiktokapi)](https://pypi.org/project/TikTokApi/) ![](https://visitor-badge.laobi.icu/badge?page_id=davidteather.TikTok-Api) [![Support Server](https://img.shields.io/discord/783108952111579166.svg?color=7289da&logo=discord&style=flat-square)](https://discord.gg/yyPhbfma6f)
 
-This api is designed to **retrieve data** TikTok. It **can not be used post or upload** content to TikTok on the behalf of a user. It has **no support any user-authenticated routes**, if you can't access it while being logged out on their website you can't access it here.
+This api is designed to **retrieve data** TikTok. It **can not be used post or upload** content to TikTok on the behalf of a user. It has **no support for any user-authenticated routes**, if you can't access it while being logged out on their website you can't access it here.
 
 ## Sponsors
 
-These sponsors have paid to be placed here and beyond that I do not have any affiliation with them, the TikTokAPI package will always be free and open-source. If you wish to be a sponsor of this project check out my [GitHub sponsors page](https://github.com/sponsors/davidteather).
+These sponsors have paid to be placed here or are my own affiliate links which I may earn a commission from, and beyond that I do not have any affiliation with them. The TikTokAPI package will always be free and open-source. If you wish to be a sponsor of this project check out my [GitHub sponsors page](https://github.com/sponsors/davidteather).
 
 <div align="center">
     <a href="https://tikapi.io/?ref=davidteather" target="_blank">
@@ -33,6 +33,14 @@ These sponsors have paid to be placed here and beyond that I do not have any aff
          <b>TikTok Captcha Solver: </b> Bypass any TikTok captcha in just two lines of code.<br> Scale your TikTok automation and get unblocked with SadCaptcha.
         </div>
     </a>
+    <br>
+    <a href="https://www.webshare.io/?referral_code=3x5812idzzzp" target="_blank">
+        <img src="https://raw.githubusercontent.com/davidteather/TikTok-Api/main/imgs/webshare.png" width="100" alt="TikTok Captcha Solver">
+        <b></b>
+        <div>
+         <b>Cheap, Reliable Proxies: </b> Supercharge your web scraping with fast, reliable proxies. Try 10 free datacenter proxies today!
+        </div>
+    </a>
 </div>
 
 ## Table of Contents
@@ -93,7 +101,8 @@ docker run -v TikTokApi --rm tiktokapi:latest python3 your_script.py
 
 ### Common Issues
 
-Please don't open an issue if you're experiencing one of these just comment if the provided solution do not work for you.
+- **EmptyResponseException** - this means TikTok is blocking the request and detects you're a bot. This can be a problem with your setup or the library itself
+  - you may need a proxy to successfuly scrape TikTok, I've made a [web scraping lesson](https://github.com/davidteather/everything-web-scraping/tree/main/002-proxies) explaining the differences of "tiers" of proxies, I've personally had success with [webshare's residential proxies](https://www.webshare.io/?referral_code=3x5812idzzzp) (affiliate link), but you might have success on their free data center IPs or a cheaper competitor.
 
 - **Browser Has no Attribute** - make sure you ran `python3 -m playwright install`, if your error persists try the [playwright-python](https://github.com/microsoft/playwright-python) quickstart guide and diagnose issues from there.
 
@@ -114,7 +123,7 @@ ms_token = os.environ.get("ms_token", None) # get your own ms_token from your co
 
 async def trending_videos():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         async for video in api.trending.videos(count=30):
             print(video)
             print(video.as_dict)

TikTokApi/stealth/js/navigator_userAgent.py

@@ -1,8 +1,16 @@
 navigator_userAgent = """
 // replace Headless references in default useragent
-const current_ua = navigator.userAgent
+const current_ua = navigator.userAgent;
 Object.defineProperty(Object.getPrototypeOf(navigator), 'userAgent', {
-    get: () => opts.navigator_user_agent || current_ua.replace('HeadlessChrome/', 'Chrome/')
-})
-
+    get: () => {
+        try {
+            if (typeof opts !== 'undefined' && opts.navigator_user_agent) {
+                return opts.navigator_user_agent;
+            }
+        } catch (error) {
+            console.warn('Error accessing opts:', error);
+        }
+        return current_ua.replace('HeadlessChrome/', 'Chrome/');
+    }
+});
 """

TikTokApi/tiktok.py

@@ -6,7 +6,7 @@
 import time
 import json
 
-from playwright.async_api import async_playwright
+from playwright.async_api import async_playwright, TimeoutError
 from urllib.parse import urlencode, quote, urlparse
 from .stealth import stealth_async
 from .helpers import random_choice
@@ -179,6 +179,7 @@ def handle_request(request):
             )
 
         await page.goto(url)
+        await page.goto(url) # hack: tiktok blocks first request not sure why
 
         session = TikTokPlaywrightSession(
             context,
@@ -363,7 +364,23 @@ async def run_fetch_script(self, url: str, headers: dict, **kwargs):
     async def generate_x_bogus(self, url: str, **kwargs):
         """Generate the X-Bogus header for a url"""
         _, session = self._get_session(**kwargs)
-        await session.page.wait_for_function("window.byted_acrawler !== undefined")
+
+        max_attempts = 5
+        attempts = 0
+        while attempts < max_attempts:
+            attempts += 1
+            try:
+                timeout_time = random.randint(5000, 20000)
+                await session.page.wait_for_function("window.byted_acrawler !== undefined", timeout=timeout_time)
+                break
+            except TimeoutError as e:
+                if attempts == max_attempts:
+                    raise TimeoutError(f"Failed to load tiktok after {max_attempts} attempts, consider using a proxy")
+
+                try_urls = ["https://www.tiktok.com/foryou", "https://www.tiktok.com", "https://www.tiktok.com/@tiktok", "https://www.tiktok.com/foryou"]
+
+                await session.page.goto(random.choice(try_urls))
+
         result = await session.page.evaluate(
             f'() => {{ return window.byted_acrawler.frontierSign("{url}") }}'
         )
@@ -452,7 +469,7 @@ async def make_request(
                 raise Exception("TikTokApi.run_fetch_script returned None")
 
             if result == "":
-                raise EmptyResponseException(result, "TikTok returned an empty response")
+                raise EmptyResponseException(result, "TikTok returned an empty response. They are detecting you're a bot, try some of these: headless=False, browser='webkit', consider using a proxy")
 
             try:
                 data = json.loads(result)

examples/comment_example.py

@@ -8,7 +8,7 @@
 
 async def get_comments():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video = api.video(id=video_id)
         count = 0
         async for comment in video.comments(count=30):

examples/hashtag_example.py

@@ -7,7 +7,7 @@
 
 async def get_hashtag_videos():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag = api.hashtag(name="funny")
         async for video in tag.videos(count=30):
             print(video)

examples/search_example.py

@@ -9,7 +9,7 @@
 
 async def search_users():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         async for user in api.search.users("david teather", count=10):
             print(user)
 

examples/sound_example.py

@@ -8,7 +8,7 @@
 
 async def sound_videos():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         async for sound in api.sound(id=sound_id).videos(count=30):
             print(sound)
             print(sound.as_dict)

examples/trending_example.py

@@ -7,7 +7,7 @@
 
 async def trending_videos():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         async for video in api.trending.videos(count=30):
             print(video)
             print(video.as_dict)

examples/user_example.py

@@ -9,7 +9,7 @@
 
 async def user_example():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         user = api.user("therock")
         user_data = await user.info()
         print(user_data)

examples/video_example.py

@@ -9,7 +9,7 @@
 
 async def get_video_example():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video = api.video(
             url="https://www.tiktok.com/@davidteathercodes/video/7074717081563942186"
         )

tests/test_comments.py

@@ -10,7 +10,7 @@
 async def test_comment_page():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video = api.video(id=video_id)
         count = 0
         async for comment in video.comments(count=100):

tests/test_hashtag.py

@@ -10,7 +10,7 @@
 async def test_hashtag_videos():
     api = TikTokApi(logging_level=logging.INFO)
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag = api.hashtag(name="funny")
         video_count = 0
         async for video in tag.videos(count=30):
@@ -23,7 +23,7 @@ async def test_hashtag_videos():
 async def test_hashtag_videos_multi_page():
     api = TikTokApi(logging_level=logging.INFO)
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag = api.hashtag(name="funny", id="5424")
         video_count = 0
         async for video in tag.videos(count=100):
@@ -36,7 +36,7 @@ async def test_hashtag_videos_multi_page():
 async def test_hashtag_info():
     api = TikTokApi(logging_level=logging.INFO)
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag = api.hashtag(name="funny")
         await tag.info()
 
@@ -48,7 +48,7 @@ async def test_hashtag_info():
 async def test_non_latin1():
     api = TikTokApi(logging_level=logging.INFO)
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag = api.hashtag(name="селфи")
         await tag.info()
 

tests/test_integration.py

@@ -8,7 +8,7 @@
 @pytest.mark.asyncio
 async def test_hashtag_videos():
     async with TikTokApi() as api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         tag_name = "funny"
         count = 0
         async for video in api.hashtag(name=tag_name).videos(count=1):

tests/test_search.py

@@ -8,7 +8,7 @@
 async def test_users_single_page():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         count = 0
         async for user in api.search.users("therock", count=10):
             count += 1
@@ -20,7 +20,7 @@ async def test_users_single_page():
 async def test_users_multi_page():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         count = 0
         async for user in api.search.users("therock", count=50):
             count += 1

tests/test_sound.py

@@ -10,7 +10,7 @@
 async def test_sound_videos():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         sound = api.sound(id=song_id)
         video_count = 0
         async for video in sound.videos(count=100):
@@ -23,9 +23,9 @@ async def test_sound_videos():
 async def test_sound_info():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         sound = api.sound(id=song_id)
         await sound.info()
         assert sound.id == song_id
         assert sound.title == "Face Off - Dwayne Johnson"
-        assert sound.duration == 45
+        assert sound.duration == 60

tests/test_trending.py

@@ -6,10 +6,10 @@
 
 
 @pytest.mark.asyncio
-async def test_user_info():
+async def test_trending():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         count = 0
         async for video in api.trending.videos(count=100):
             count += 1

tests/test_user.py

@@ -13,7 +13,7 @@
 async def test_user_info():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         user = api.user(username=username)
         await user.info()
 
@@ -25,7 +25,7 @@ async def test_user_info():
 async def test_user_videos():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         user = api.user(username=username, sec_uid=sec_uid, user_id=user_id)
 
         count = 0
@@ -38,7 +38,7 @@ async def test_user_videos():
 async def test_user_likes():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         user = api.user(
             username="publicliketest",
             sec_uid="MS4wLjABAAAAHjhwCIwmvzVZfRrDAZ2aZy74LciLnoyaPfM2rrX9N7bwbWMFuwTFG4YrByYvsH5c",
@@ -54,7 +54,7 @@ async def test_user_likes():
 async def test_user_playlists():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         user = api.user(username="mrbeast")
 
         count = 0

tests/test_video.py

@@ -9,7 +9,7 @@
 async def test_video_id_from_url():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
 
         expected_id = "7074717081563942186"
         video = api.video(
@@ -28,7 +28,7 @@ async def test_video_id_from_url():
 async def test_video_info():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video_id = "7074717081563942186"
         video = api.video(
             url="https://www.tiktok.com/@davidteathercodes/video/7074717081563942186"
@@ -45,7 +45,7 @@ async def test_video_bytes():
     pytest.skip("Not implemented yet")
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video_id = "7107272719166901550"
         video = api.video(id=video_id)
 
@@ -57,7 +57,7 @@ async def test_video_bytes():
 async def test_related_videos():
     api = TikTokApi()
     async with api:
-        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3, browser=os.getenv("TIKTOK_BROWSER", None))
         video_id = "7107272719166901550"
         video = api.video(id=video_id)
         count = 0