Skip to content

Instantly share code, notes, and snippets.

@rednafi

rednafi/getUrls.md

Last active May 20, 2023 07:10
Show Gist options
  • Save rednafi/91117dc75f2dac1ab17065a60ffab185 to your computer and use it in GitHub Desktop.
Save rednafi/91117dc75f2dac1ab17065a60ffab185 to your computer and use it in GitHub Desktop.
Download ZIP
Submit URLs for indexing via googleapis
Raw
getUrls.md

Prerequisites

https://developers.google.com/search/apis/indexing-api/v3/quickstart

Collect the URLs from sitemap

Run this directly from the browser console:

async function getSitemapUrls(sitemapUrl) {
  try {
    const response = await fetch(sitemapUrl);
    if (!response.ok) {
      throw new Error(
        `Failed to load sitemap (${response.status}): ${sitemapUrl}`
      );
    }
    const sitemapXml = await response.text();
    const parser = new DOMParser();
    const xmlDoc = parser.parseFromString(sitemapXml, "application/xml");
    const urlElements = xmlDoc.getElementsByTagName("url");
    const urls = Array.from(urlElements).map((urlElement) => {
      return urlElement.getElementsByTagName("loc")[0].textContent;
    });
    return urls;
  } catch (error) {
    throw error;
  }
}

// Usage example:
const sitemapUrl = "https://rednafi.com/sitemap.xml";

(async () => {
  try {
    const urls = await getSitemapUrls(sitemapUrl);
    console.log(urls);
  } catch (error) {
    console.error(error);
  }
})();

Output:

[
    "https://rednafi.com/misc/fixed_time_task_scheduling_with_at/",
    "https://rednafi.com/tags/javascript/",
    "https://rednafi.com/misc/",
    "https://rednafi.com/tags/networking/",
    "https://rednafi.com/",
    "https://rednafi.com/tags/shell/",
    "https://rednafi.com/tags/",
    "https://rednafi.com/tags/django/",
    "https://rednafi.com/tags/python/",
    "https://rednafi.com/python/",
    "https://rednafi.com/python/sort_by_a_custom_sequence_in_django/",
    "https://rednafi.com/tags/github/",
    "https://rednafi.com/javascript/",
    "https://rednafi.com/javascript/periodic_readme_updates_with_gh_actions/",
    "https://rednafi.com/misc/associative_arrays_in_bash/",
    "https://rednafi.com/tags/til/",
    "https://rednafi.com/python/deduplicate_iterables_while_preserving_order/",
    "https://rednafi.com/misc/process_substitution_in_bash/",
    "https://rednafi.com/misc/dynamic_menu_with_select_in_bash/",
    "https://rednafi.com/misc/terminal_text_formatting_with_tput/",
    "https://rednafi.com/javascript/exploring_observable_notebooks/",
    "https://rednafi.com/python/server_sent_events/",
    "https://rednafi.com/misc/tinkering_with_unix_domain_socket/",
    "https://rednafi.com/python/multithreaded_socket_server_signal_handling/",
    "https://rednafi.com/tags/database/",
    "https://rednafi.com/python/switch_between_multiple_datastreams/",
    "https://rednafi.com/python/skip_first_part_of_an_iterable/",
    "https://rednafi.com/python/pause_and_resume_a_socket_server/",
    "https://rednafi.com/python/django_and_jupyter_notebook/",
    "https://rednafi.com/python/manipulate_text_with_django_query_expression/",
    "https://rednafi.com/python/tqdm_progressbar_with_concurrent_futures/",
    "https://rednafi.com/misc/colon_command_in_shell_scripts/",
    "https://rednafi.com/python/faster_bulk_update_in_django/",
    "https://rednafi.com/python/install_python_with_asdf/",
    "https://rednafi.com/python/save_with_update_fields_in_django/",
    "https://rednafi.com/tags/aws/",
    "https://rednafi.com/python/logging_quirks_in_lambda_environment/",
    "https://rednafi.com/python/outage_caused_by_eager_loading_file/",
    "https://rednafi.com/tags/incident-post-mortem/",
    "https://rednafi.com/misc/audit_commit_messages_on_github/",
    "https://rednafi.com/misc/to_quote_or_not_to_quote/",
    "https://rednafi.com/misc/return_values_from_a_shell_function/",
    "https://rednafi.com/tags/api/",
    "https://rednafi.com/python/verify_webhook_origin/",
    "https://rednafi.com/python/recipes_from_python_sqlite_docs/",
    "https://rednafi.com/python/use_urlsplit_over_urlparse/",
    "https://rednafi.com/databases/",
    "https://rednafi.com/databases/random_choice_in_sqlite/",
    "https://rednafi.com/tags/sql/",
    "https://rednafi.com/python/exitstack/",
    "https://rednafi.com/python/compose_multiple_levels_of_pytest_fixtures/",
    "https://rednafi.com/tags/testing/",
    "https://rednafi.com/python/patch_where_the_object_is_used/",
    "https://rednafi.com/python/partially_assert_callable_arguments/",
    "https://rednafi.com/tags/git/",
    "https://rednafi.com/misc/when_to_use_git_pull_rebase/",
    "https://rednafi.com/python/apply_constraint_with_assert/",
    "https://rednafi.com/misc/automerge_dependabot_prs_on_github/",
    "https://rednafi.com/python/stream_process_a_csv_file/",
    "https://rednafi.com/python/django_bulk_operation_with_process_pool/",
    "https://rednafi.com/python/read_s3_file_in_memory/",
    "https://rednafi.com/misc/distil_git_logs_attached_to_a_file/",
    "https://rednafi.com/python/operators_itemgetter/",
    "https://rednafi.com/typescript/guard_clauses_and_never_type/",
    "https://rednafi.com/tags/typescript/",
    "https://rednafi.com/typescript/",
    "https://rednafi.com/tags/typing/",
    "https://rednafi.com/misc/health_check_a_server_with_nohup/",
    "https://rednafi.com/python/return_json_error_payload_in_drf/",
    "https://rednafi.com/python/decouple_with_generators/",
    "https://rednafi.com/python/preallocated_list/",
    "https://rednafi.com/random/in_favor_of_sentence_case/",
    "https://rednafi.com/tags/meta/",
    "https://rednafi.com/random/",
    "https://rednafi.com/python/disallow_large_file_download/",
    "https://rednafi.com/python/declaratively_transform_dataclass_fields/",
    "https://rednafi.com/python/caching_connection_objects/",
    "https://rednafi.com/python/how_not_to_run_a_script/",
    "https://rednafi.com/python/mocking_datetime_objects/",
    "https://rednafi.com/python/declarative_payloads_with_typedict/",
    "https://rednafi.com/python/parametrized_fixtures_in_pytest/",
    "https://rednafi.com/python/modify_iterables_while_iterating/",
    "https://rednafi.com/python/github_action_template_python/",
    "https://rednafi.com/python/self_type/",
    "https://rednafi.com/python/patch_with_pytest_fixture/",
    "https://rednafi.com/python/type_guard/",
    "https://rednafi.com/python/why_noreturn_type_exists/",
    "https://rednafi.com/python/add_attributes_to_enum_members/",
    "https://rednafi.com/python/internals_of_functools_wraps/",
    "https://rednafi.com/python/limit_concurrency_with_semaphore/",
    "https://rednafi.com/python/amphibian_decorators/",
    "https://rednafi.com/python/go_rusty_with_exception_handling/",
    "https://rednafi.com/python/variance_of_generic_types/",
    "https://rednafi.com/python/create_sub_dict/",
    "https://rednafi.com/python/early_bound_function_defaults/",
    "https://rednafi.com/python/use_assertis_to_check_literal_booleans/",
    "https://rednafi.com/python/static_typing_decorators/",
    "https://rednafi.com/python/inspect_docstring_with_pydoc/",
    "https://rednafi.com/python/check_is_a_power_of_two/",
    "https://rednafi.com/python/uniform_error_response_in_drf/",
    "https://rednafi.com/python/difference_between_typevar_and_union/",
    "https://rednafi.com/python/lru_cache_on_methods/",
    "https://rednafi.com/python/text_cropping_with_textwrap_shorten/",
    "https://rednafi.com/python/string_interning/",
    "https://rednafi.com/python/structural_subtyping/",
    "https://rednafi.com/python/attribute_delegation_in_composition/",
    "https://rednafi.com/python/access_classmethod_like_property/",
    "https://rednafi.com/misc/do_not_add_extenstions_to_bash_executables/",
    "https://rednafi.com/python/use_init_subclass_hook_to_validate_subclasses/",
    "https://rednafi.com/python/tqdm_with_multiprocessing/",
    "https://rednafi.com/python/use_daemon_threads_to_test_infinite_loop/",
    "https://rednafi.com/misc/use_command_v_over_which/",
    "https://rednafi.com/misc/write_git_commit_messages_properly/",
    "https://rednafi.com/python/functools_partial_flattens_nestings_automatically/",
    "https://rednafi.com/misc/use_curly_braces_while_pasting_shell_commands/",
    "https://rednafi.com/python/config_management_with_pydantic/",
    "https://rednafi.com/python/mixins/",
    "https://rednafi.com/python/metaclasses/",
    "https://rednafi.com/python/proxy_pattern/",
    "https://rednafi.com/python/redis_cache/",
    "https://rednafi.com/python/decorators/",
    "https://rednafi.com/python/concurrent_futures/",
    "https://rednafi.com/python/pathlib/",
    "https://rednafi.com/python/pre_commit/",
    "https://rednafi.com/python/singledispatch/",
    "https://rednafi.com/python/contextmanager/",
    "https://rednafi.com/python/dataclasses/",
    "https://rednafi.com/archives/",
    "https://rednafi.com/categories/",
    "https://rednafi.com/contacts/",
    "https://rednafi.com/search/"
]

Make API request

import request from "request";
import { google } from "googleapis";
import data from "./../../../../Downloads/peaceful-campus-360402-b89a8b824b44.json" assert {type: "json"};

const jwtClient = new google.auth.JWT(
  data.client_email,
  null,
  data.private_key,
  ["https://www.googleapis.com/auth/indexing"],
  null
);

const urls = [
  "https://rednafi.com/misc/fixed_time_task_scheduling_with_at/",
  "https://rednafi.com/tags/javascript/",
  "https://rednafi.com/misc/",
  "https://rednafi.com/tags/networking/",
  "https://rednafi.com/",
  "https://rednafi.com/tags/shell/",
  "https://rednafi.com/tags/",
  "https://rednafi.com/tags/django/",
  "https://rednafi.com/tags/python/",
  "https://rednafi.com/python/",
  "https://rednafi.com/python/sort_by_a_custom_sequence_in_django/",
  "https://rednafi.com/tags/github/",
  "https://rednafi.com/javascript/",
  "https://rednafi.com/javascript/periodic_readme_updates_with_gh_actions/",
  "https://rednafi.com/misc/associative_arrays_in_bash/",
  "https://rednafi.com/tags/til/",
  "https://rednafi.com/python/deduplicate_iterables_while_preserving_order/",
  "https://rednafi.com/misc/process_substitution_in_bash/",
  "https://rednafi.com/misc/dynamic_menu_with_select_in_bash/",
  "https://rednafi.com/misc/terminal_text_formatting_with_tput/",
  "https://rednafi.com/javascript/exploring_observable_notebooks/",
  "https://rednafi.com/python/server_sent_events/",
  "https://rednafi.com/misc/tinkering_with_unix_domain_socket/",
  "https://rednafi.com/python/multithreaded_socket_server_signal_handling/",
  "https://rednafi.com/tags/database/",
  "https://rednafi.com/python/switch_between_multiple_datastreams/",
  "https://rednafi.com/python/skip_first_part_of_an_iterable/",
  "https://rednafi.com/python/pause_and_resume_a_socket_server/",
  "https://rednafi.com/python/django_and_jupyter_notebook/",
  "https://rednafi.com/python/manipulate_text_with_django_query_expression/",
  "https://rednafi.com/python/tqdm_progressbar_with_concurrent_futures/",
  "https://rednafi.com/misc/colon_command_in_shell_scripts/",
  "https://rednafi.com/python/faster_bulk_update_in_django/",
  "https://rednafi.com/python/install_python_with_asdf/",
  "https://rednafi.com/python/save_with_update_fields_in_django/",
  "https://rednafi.com/tags/aws/",
  "https://rednafi.com/python/logging_quirks_in_lambda_environment/",
  "https://rednafi.com/python/outage_caused_by_eager_loading_file/",
  "https://rednafi.com/tags/incident-post-mortem/",
  "https://rednafi.com/misc/audit_commit_messages_on_github/",
  "https://rednafi.com/misc/to_quote_or_not_to_quote/",
  "https://rednafi.com/misc/return_values_from_a_shell_function/",
  "https://rednafi.com/tags/api/",
  "https://rednafi.com/python/verify_webhook_origin/",
  "https://rednafi.com/python/recipes_from_python_sqlite_docs/",
  "https://rednafi.com/python/use_urlsplit_over_urlparse/",
  "https://rednafi.com/databases/",
  "https://rednafi.com/databases/random_choice_in_sqlite/",
  "https://rednafi.com/tags/sql/",
  "https://rednafi.com/python/exitstack/",
  "https://rednafi.com/python/compose_multiple_levels_of_pytest_fixtures/",
  "https://rednafi.com/tags/testing/",
  "https://rednafi.com/python/patch_where_the_object_is_used/",
  "https://rednafi.com/python/partially_assert_callable_arguments/",
  "https://rednafi.com/tags/git/",
  "https://rednafi.com/misc/when_to_use_git_pull_rebase/",
  "https://rednafi.com/python/apply_constraint_with_assert/",
  "https://rednafi.com/misc/automerge_dependabot_prs_on_github/",
  "https://rednafi.com/python/stream_process_a_csv_file/",
  "https://rednafi.com/python/django_bulk_operation_with_process_pool/",
  "https://rednafi.com/python/read_s3_file_in_memory/",
  "https://rednafi.com/misc/distil_git_logs_attached_to_a_file/",
  "https://rednafi.com/python/operators_itemgetter/",
  "https://rednafi.com/typescript/guard_clauses_and_never_type/",
  "https://rednafi.com/tags/typescript/",
  "https://rednafi.com/typescript/",
  "https://rednafi.com/tags/typing/",
  "https://rednafi.com/misc/health_check_a_server_with_nohup/",
  "https://rednafi.com/python/return_json_error_payload_in_drf/",
  "https://rednafi.com/python/decouple_with_generators/",
  "https://rednafi.com/python/preallocated_list/",
  "https://rednafi.com/random/in_favor_of_sentence_case/",
  "https://rednafi.com/tags/meta/",
  "https://rednafi.com/random/",
  "https://rednafi.com/python/disallow_large_file_download/",
  "https://rednafi.com/python/declaratively_transform_dataclass_fields/",
  "https://rednafi.com/python/caching_connection_objects/",
  "https://rednafi.com/python/how_not_to_run_a_script/",
  "https://rednafi.com/python/mocking_datetime_objects/",
  "https://rednafi.com/python/declarative_payloads_with_typedict/",
  "https://rednafi.com/python/parametrized_fixtures_in_pytest/",
  "https://rednafi.com/python/modify_iterables_while_iterating/",
  "https://rednafi.com/python/github_action_template_python/",
  "https://rednafi.com/python/self_type/",
  "https://rednafi.com/python/patch_with_pytest_fixture/",
  "https://rednafi.com/python/type_guard/",
  "https://rednafi.com/python/why_noreturn_type_exists/",
  "https://rednafi.com/python/add_attributes_to_enum_members/",
  "https://rednafi.com/python/internals_of_functools_wraps/",
  "https://rednafi.com/python/limit_concurrency_with_semaphore/",
  "https://rednafi.com/python/amphibian_decorators/",
  "https://rednafi.com/python/go_rusty_with_exception_handling/",
  "https://rednafi.com/python/variance_of_generic_types/",
  "https://rednafi.com/python/create_sub_dict/",
  "https://rednafi.com/python/early_bound_function_defaults/",
  "https://rednafi.com/python/use_assertis_to_check_literal_booleans/",
  "https://rednafi.com/python/static_typing_decorators/",
  "https://rednafi.com/python/inspect_docstring_with_pydoc/",
  "https://rednafi.com/python/check_is_a_power_of_two/",
  "https://rednafi.com/python/uniform_error_response_in_drf/",
  "https://rednafi.com/python/difference_between_typevar_and_union/",
  "https://rednafi.com/python/lru_cache_on_methods/",
  "https://rednafi.com/python/text_cropping_with_textwrap_shorten/",
  "https://rednafi.com/python/string_interning/",
  "https://rednafi.com/python/structural_subtyping/",
  "https://rednafi.com/python/attribute_delegation_in_composition/",
  "https://rednafi.com/python/access_classmethod_like_property/",
  "https://rednafi.com/misc/do_not_add_extenstions_to_bash_executables/",
  "https://rednafi.com/python/use_init_subclass_hook_to_validate_subclasses/",
  "https://rednafi.com/python/tqdm_with_multiprocessing/",
  "https://rednafi.com/python/use_daemon_threads_to_test_infinite_loop/",
  "https://rednafi.com/misc/use_command_v_over_which/",
  "https://rednafi.com/misc/write_git_commit_messages_properly/",
  "https://rednafi.com/python/functools_partial_flattens_nestings_automatically/",
  "https://rednafi.com/misc/use_curly_braces_while_pasting_shell_commands/",
  "https://rednafi.com/python/config_management_with_pydantic/",
  "https://rednafi.com/python/mixins/",
  "https://rednafi.com/python/metaclasses/",
  "https://rednafi.com/python/proxy_pattern/",
  "https://rednafi.com/python/redis_cache/",
  "https://rednafi.com/python/decorators/",
  "https://rednafi.com/python/concurrent_futures/",
  "https://rednafi.com/python/pathlib/",
  "https://rednafi.com/python/pre_commit/",
  "https://rednafi.com/python/singledispatch/",
  "https://rednafi.com/python/contextmanager/",
  "https://rednafi.com/python/dataclasses/",
  "https://rednafi.com/archives/",
  "https://rednafi.com/categories/",
  "https://rednafi.com/contacts/",
  "https://rednafi.com/search/"
]

jwtClient.authorize(function (err, tokens) {
  if (err) {
    console.log(err);
    return;
  }

  for (let url of urls) {
    let options = {
      url: "https://indexing.googleapis.com/v3/urlNotifications:publish",
      method: "POST",
      // Your options, which must include the Content-Type and auth headers
      headers: {
        "Content-Type": "application/json",
      },
      auth: { bearer: tokens.access_token },
      // Define contents here. The structure of the content is described in the next step.
      json: {
        url: url,
        type: "URL_UPDATED",
      },
    };

    request(options, function (error, response, body) {
      // Handle the response
      console.log(body);
    });
  }
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment