From 100e262aa8923203809e33ad4e2977cec4c3bb4c Mon Sep 17 00:00:00 2001 From: ErdemOzgen <14043035+ErdemOzgen@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:37:12 +0300 Subject: [PATCH] general refactoring --- content/CyberSecurity/Index.md | 5 +- .../Untitled.md} | 0 content/Data Marts.md | 23 ----- content/Modern Data Stack.md | 82 ------------------ .../Dependency Injection.md | 0 content/SoftwareEngineering/Leetcode/links.md | 2 + .../ProgramingLang/Blogs/List of blogs.md | 0 .../ProgramingLang/Index.md | 0 .../golang/Concurrency Testing in Golang.md | 0 .../ProgramingLang/golang/Go By example.md | 0 .../golang/Go Common Mistakes.md | 2 +- .../Golang Web Programing.md} | 0 ...ang - Asterisk and Ampersand Cheatsheet.md | 0 .../golang/Golang Cobra and Viper.md | 0 .../ProgramingLang/golang/Golang context.md | 0 .../golang/Learn Go with Tests.md | 0 .../python/Django-Cookiecutter.md | 0 .../ProgramingLang/python/Django.md | 0 .../ProgramingLang/python/Poetry.md | 0 .../ProgramingLang/python/Pyarmor.md | 0 .../ProgramingLang/python/Selenium.md | 0 .../examples in golang.md | 0 .../hexagonal architecture.md | 0 content/Untitled 1.canvas | 1 - content/Untitled 2.canvas | 1 - content/Untitled.canvas | 1 - .../Pasted image 20240929121007.png | Bin .../Pasted image 20240929121912.png | Bin .../Pasted image 20240929122545.png | Bin .../Screenshot 2024-11-22 at 13.07.15.png | Bin .../Screenshot 2024-11-22 at 13.20.08.png | Bin .../Screenshot 2024-11-22 at 13.23.21.png | Bin .../Screenshot 2024-11-22 at 14.30.18.png | Bin 33 files changed, 7 insertions(+), 110 deletions(-) rename content/{ProgramingLang/golang/Go Web Development/Golang Web Programing.md => CyberSecurity/Untitled.md} (100%) delete mode 100644 content/Data Marts.md delete mode 100644 content/Modern Data Stack.md rename content/{ => SoftwareEngineering}/Dependency Injection.md (100%) create mode 100644 content/SoftwareEngineering/Leetcode/links.md rename content/{ => SoftwareEngineering}/ProgramingLang/Blogs/List of blogs.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/Index.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Concurrency Testing in Golang.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Go By example.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Go Common Mistakes.md (99%) rename content/{medium article ideas.md => SoftwareEngineering/ProgramingLang/golang/Go Web Development/Golang Web Programing.md} (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Golang - Asterisk and Ampersand Cheatsheet.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Golang Cobra and Viper.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Golang context.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/golang/Learn Go with Tests.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/python/Django-Cookiecutter.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/python/Django.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/python/Poetry.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/python/Pyarmor.md (100%) rename content/{ => SoftwareEngineering}/ProgramingLang/python/Selenium.md (100%) rename content/{ => SoftwareEngineering}/examples in golang.md (100%) rename content/{ => SoftwareEngineering}/hexagonal architecture.md (100%) delete mode 100644 content/Untitled 1.canvas delete mode 100644 content/Untitled 2.canvas delete mode 100644 content/Untitled.canvas rename content/{ => imgs}/Pasted image 20240929121007.png (100%) rename content/{ => imgs}/Pasted image 20240929121912.png (100%) rename content/{ => imgs}/Pasted image 20240929122545.png (100%) rename content/{ => imgs}/Screenshot 2024-11-22 at 13.07.15.png (100%) rename content/{ => imgs}/Screenshot 2024-11-22 at 13.20.08.png (100%) rename content/{ => imgs}/Screenshot 2024-11-22 at 13.23.21.png (100%) rename content/{ => imgs}/Screenshot 2024-11-22 at 14.30.18.png (100%) diff --git a/content/CyberSecurity/Index.md b/content/CyberSecurity/Index.md index be72db53d..c0bdb6a28 100644 --- a/content/CyberSecurity/Index.md +++ b/content/CyberSecurity/Index.md @@ -2,4 +2,7 @@ tags: - "#cyber" --- -#cyber \ No newline at end of file +#cyber +#sourcecodeaudit +#devsecops +#oswe diff --git a/content/ProgramingLang/golang/Go Web Development/Golang Web Programing.md b/content/CyberSecurity/Untitled.md similarity index 100% rename from content/ProgramingLang/golang/Go Web Development/Golang Web Programing.md rename to content/CyberSecurity/Untitled.md diff --git a/content/Data Marts.md b/content/Data Marts.md deleted file mode 100644 index 086040906..000000000 --- a/content/Data Marts.md +++ /dev/null @@ -1,23 +0,0 @@ -A data mart is a subset of a data warehouse that is usually oriented to a specific business line or team. Whereas data warehouses have enterprise-wide depth, data marts are often smaller sections of data warehouses segmented for specific uses. Here are some key points about data marts: - -1. **Purpose-Oriented**: Data marts are designed to meet the specific needs of a particular group of users, such as a department within a company, like sales, finance, or marketing. They contain data relevant to that group. - -2. **Scope and Size**: They are smaller in scope and size compared to data warehouses. This makes them easier to implement and manage. - -3. **Data Sources**: Data marts can draw data from a wide range of sources. They may pull data from internal systems like ERP and CRM, as well as external data sources. - -4. **Performance**: Because of their smaller size and focused nature, data marts can improve query performance. They enable users to access and analyze relevant data more quickly and efficiently than sifting through a larger data warehouse. - -5. **Types of Data Marts**: There are two main types: independent data marts, which are created without a data warehouse and rely solely on data from source systems; and dependent data marts, which are created from an existing data warehouse. - -6. **Implementation**: Implementing a data mart is generally faster and less costly than a full data warehouse because of its limited scope. - -7. **Business Intelligence**: Data marts are often used in business intelligence (BI) applications, where they provide the data for reporting and analysis tools. - -8. **Customization**: They can be customized to fit the specific needs of different user groups within an organization, providing more relevant and tailored data for analysis. - -In summary, data marts are a focused and scaled-down version of data warehouses, designed to provide specific groups within an organization with the data they need for analysis and decision-making. - -https://community.databricks.com/t5/data-engineering/how-to-build-data-warehouses-and-data-marts-with-python/td-p/42843 - - diff --git a/content/Modern Data Stack.md b/content/Modern Data Stack.md deleted file mode 100644 index 3427c8495..000000000 --- a/content/Modern Data Stack.md +++ /dev/null @@ -1,82 +0,0 @@ -# Modern Data Stack - -Last updated Dec 7, 2023 - -- [Dataquestion](https://www.ssp.sh/brain/tags/dataquestion/) - -Table of Contents - -1. [Why the Modern Data Stack?](https://www.ssp.sh/brain/modern-data-stack/#why-the-modern-data-stack) -2. [Integrating with [[Dagster]]](https://www.ssp.sh/brain/modern-data-stack/#integrating-with-dagster) -3. [A comment I made on Social](https://www.ssp.sh/brain/modern-data-stack/#a-comment-i-made-on-social) -4. [Further Links](https://www.ssp.sh/brain/modern-data-stack/#further-links) -5. [[[Counter Arguments against Modern Data Stack]]](https://www.ssp.sh/brain/modern-data-stack/#counter-arguments-against-modern-data-stack) - -The Modern Data Stack (MDS) comprises a suite of open-source tools designed for end-to-end analytics. This includes data ingestion, transformation, machine learning, and integration into a columnar data warehouse or lake solution, all complemented by an analytics BI dashboard backend. The stack’s versatility allows extensions for data quality, data cataloging, and more. - -MDS aims to enable data insights using the best-suited tools for each process. It’s worth noting that “Modern Data Stack” is a relatively new term, with its definition still evolving. - -> Synonym Names -> A burgeoning term, [ngods (new generation open-source data stack)](https://www.ssp.sh/brain/ngods-new-generation-open-source-data-stack), has emerged. Previously, I’ve referred to this concept as the Open Data Stack Project. Additionally, Dagster introduced the term DataStack 2.0 in a [recent blog post](https://dagster.io/blog/evolution-iq-case-study). [Open Data Stack](https://www.ssp.sh/brain/open-data-stack) is my own definition of it. - -> Closed Source vs Open Source -> Closed Source examples: dbt, Looker, Snowflake, Fivetran, Hightouch, Census -> Open Source alternatives: airbyte, dbt, dagster, Superset, Reverse-ETL? - -> Modern Data Stack on a Laptop -> [DuckDB: Modern Data Stack in a Box](https://duckdb.org/2022/10/12/modern-data-stack-in-a-box.html) - -[ - -## # Why the Modern Data Stack? - -](https://www.ssp.sh/brain/modern-data-stack/#why-the-modern-data-stack) - -A perspective from [Reddit](https://www.reddit.com/r/dataengineering/comments/12acdrk/comment/jes4pr8/?utm_source=share&utm_medium=web2x&context=3) highlights the shift in data warehousing and analytics. It underscores the reduced need for extensive teams and infrastructure, thanks to new tools that streamline data management and reporting. Particularly for small and mid-sized companies, MDS offers a competitive edge in data handling, allowing even a single data engineer to manage vast datasets efficiently. - ---- - -A notable article discussing Lakehouse, Metrics Layer, and Clickhouse: -[The Next Cloud Data Platform | Greylock](https://greylock.com/greymatter/the-next-cloud-data-platform/) -![](https://www.ssp.sh/brain/Pasted%20image%2020220527120559.png) - -[](https://www.ssp.sh/brain/modern-data-stack/#integrating-with-dagster) - -## [# Integrating with](https://www.ssp.sh/brain/modern-data-stack/#integrating-with-dagster) Dagster - -The downside of MDS is the unbundling of Bundling vs Unbundling- Monolith Data vs Microservices, but Dagster helps integrate the full data stack together: -![](https://www.ssp.sh/brain/Pasted%20image%2020220428103513.png) -![](https://www.ssp.sh/brain/Pasted%20image%2020220428103632.png) -Dagster elevates the Modern Data Stack: -![](https://www.ssp.sh/brain/Pasted%20image%2020220428103934.png) -![](https://www.ssp.sh/brain/Pasted%20image%2020220428103939.png) -Explore more about its power with Dagster and [Data Assets](https://www.ssp.sh/brain/data-assets). - -[ - -## # A comment I made on Social - -](https://www.ssp.sh/brain/modern-data-stack/#a-comment-i-made-on-social) - -I often ponder over the ideal tools for a data stack. My preference leans toward a [Cloud Data Warehouse](https://www.sspaeti.com/blog/olap-whats-coming-next/#cloud-data-warehouses) such as [](https://www.firebolt.io/)Firebolt, [Snowflake](https://www.snowflake.com/), [BigQuery](https://cloud.google.com/bigquery/), [Redshift](https://aws.amazon.com/redshift/), or [Synapse](https://azure.microsoft.com/en-us/services/synapse-analytics/), as a starting point. - -The journey typically begins with [Airbyte](https://airbyte.com/) for data integration, followed by SQL-based transformation with [dbt](https://www.getdbt.com/). Orchestrating the processes in [Python](https://www.sspaeti.com/blog/business-intelligence-meets-data-engineering/#8220use-python-and-sql-if-possible8221) with tools like [dagster](https://ask.astorik.com/c/resources-sspaeti/what-are-common-alternatives-to-apache-airflow) is crucial. - -From there, I would integrate additional open-source tools based on specific needs: [Spark](https://spark.apache.org/) for processing, [Delta Lake](https://delta.io/) for data lake formatting and ACID Transactions, [Amundsen](http://amundsen.io/) for data cataloging, and [Great Expectation](https://greatexpectations.io/) for data quality, among others. For smaller projects, [DuckDB](https://duckdb.org/) is suitable for local [OLAP](https://www.ssp.sh/brain/olap) scenarios, while [Kubernetes](https://kubernetes.io/) and DevOps provide scalability. - -For teams without data engineering resources, closed-source options like [Ascend](https://www.ascend.io/) or [Foundry](https://www.palantir.com/platforms/foundry/) are viable alternatives. - -Feel free to reach out for further discussion or clarifications. - -[ - -## # Further Links - -](https://www.ssp.sh/brain/modern-data-stack/#further-links) - -- [The next layer of the modern data stack](https://www.getdbt.com/blog/next-layer-of-the-modern-data-stack/) -- [What Is the Modern Data Stack? | Fivetran](https://www.fivetran.com/blog/what-is-the-modern-data-stack) - -[](https://www.ssp.sh/brain/modern-data-stack/#counter-arguments-against-modern-data-stack) - -## [#](https://www.ssp.sh/brain/modern-data-stack/#counter-arguments-against-modern-data-stack) Counter Arguments against Modern Data Stack \ No newline at end of file diff --git a/content/Dependency Injection.md b/content/SoftwareEngineering/Dependency Injection.md similarity index 100% rename from content/Dependency Injection.md rename to content/SoftwareEngineering/Dependency Injection.md diff --git a/content/SoftwareEngineering/Leetcode/links.md b/content/SoftwareEngineering/Leetcode/links.md new file mode 100644 index 000000000..a72d6628d --- /dev/null +++ b/content/SoftwareEngineering/Leetcode/links.md @@ -0,0 +1,2 @@ + +https://www.youtube.com/watch?v=lvO88XxNAzs \ No newline at end of file diff --git a/content/ProgramingLang/Blogs/List of blogs.md b/content/SoftwareEngineering/ProgramingLang/Blogs/List of blogs.md similarity index 100% rename from content/ProgramingLang/Blogs/List of blogs.md rename to content/SoftwareEngineering/ProgramingLang/Blogs/List of blogs.md diff --git a/content/ProgramingLang/Index.md b/content/SoftwareEngineering/ProgramingLang/Index.md similarity index 100% rename from content/ProgramingLang/Index.md rename to content/SoftwareEngineering/ProgramingLang/Index.md diff --git a/content/ProgramingLang/golang/Concurrency Testing in Golang.md b/content/SoftwareEngineering/ProgramingLang/golang/Concurrency Testing in Golang.md similarity index 100% rename from content/ProgramingLang/golang/Concurrency Testing in Golang.md rename to content/SoftwareEngineering/ProgramingLang/golang/Concurrency Testing in Golang.md diff --git a/content/ProgramingLang/golang/Go By example.md b/content/SoftwareEngineering/ProgramingLang/golang/Go By example.md similarity index 100% rename from content/ProgramingLang/golang/Go By example.md rename to content/SoftwareEngineering/ProgramingLang/golang/Go By example.md diff --git a/content/ProgramingLang/golang/Go Common Mistakes.md b/content/SoftwareEngineering/ProgramingLang/golang/Go Common Mistakes.md similarity index 99% rename from content/ProgramingLang/golang/Go Common Mistakes.md rename to content/SoftwareEngineering/ProgramingLang/golang/Go Common Mistakes.md index c37f96287..789808b73 100644 --- a/content/ProgramingLang/golang/Go Common Mistakes.md +++ b/content/SoftwareEngineering/ProgramingLang/golang/Go Common Mistakes.md @@ -6,7 +6,7 @@ tags: --- - [[Go By example]] -[[ProgramingLang/Index|Index]] +[[SoftwareEngineering/ProgramingLang/Index|Index]] diff --git a/content/medium article ideas.md b/content/SoftwareEngineering/ProgramingLang/golang/Go Web Development/Golang Web Programing.md similarity index 100% rename from content/medium article ideas.md rename to content/SoftwareEngineering/ProgramingLang/golang/Go Web Development/Golang Web Programing.md diff --git a/content/ProgramingLang/golang/Golang - Asterisk and Ampersand Cheatsheet.md b/content/SoftwareEngineering/ProgramingLang/golang/Golang - Asterisk and Ampersand Cheatsheet.md similarity index 100% rename from content/ProgramingLang/golang/Golang - Asterisk and Ampersand Cheatsheet.md rename to content/SoftwareEngineering/ProgramingLang/golang/Golang - Asterisk and Ampersand Cheatsheet.md diff --git a/content/ProgramingLang/golang/Golang Cobra and Viper.md b/content/SoftwareEngineering/ProgramingLang/golang/Golang Cobra and Viper.md similarity index 100% rename from content/ProgramingLang/golang/Golang Cobra and Viper.md rename to content/SoftwareEngineering/ProgramingLang/golang/Golang Cobra and Viper.md diff --git a/content/ProgramingLang/golang/Golang context.md b/content/SoftwareEngineering/ProgramingLang/golang/Golang context.md similarity index 100% rename from content/ProgramingLang/golang/Golang context.md rename to content/SoftwareEngineering/ProgramingLang/golang/Golang context.md diff --git a/content/ProgramingLang/golang/Learn Go with Tests.md b/content/SoftwareEngineering/ProgramingLang/golang/Learn Go with Tests.md similarity index 100% rename from content/ProgramingLang/golang/Learn Go with Tests.md rename to content/SoftwareEngineering/ProgramingLang/golang/Learn Go with Tests.md diff --git a/content/ProgramingLang/python/Django-Cookiecutter.md b/content/SoftwareEngineering/ProgramingLang/python/Django-Cookiecutter.md similarity index 100% rename from content/ProgramingLang/python/Django-Cookiecutter.md rename to content/SoftwareEngineering/ProgramingLang/python/Django-Cookiecutter.md diff --git a/content/ProgramingLang/python/Django.md b/content/SoftwareEngineering/ProgramingLang/python/Django.md similarity index 100% rename from content/ProgramingLang/python/Django.md rename to content/SoftwareEngineering/ProgramingLang/python/Django.md diff --git a/content/ProgramingLang/python/Poetry.md b/content/SoftwareEngineering/ProgramingLang/python/Poetry.md similarity index 100% rename from content/ProgramingLang/python/Poetry.md rename to content/SoftwareEngineering/ProgramingLang/python/Poetry.md diff --git a/content/ProgramingLang/python/Pyarmor.md b/content/SoftwareEngineering/ProgramingLang/python/Pyarmor.md similarity index 100% rename from content/ProgramingLang/python/Pyarmor.md rename to content/SoftwareEngineering/ProgramingLang/python/Pyarmor.md diff --git a/content/ProgramingLang/python/Selenium.md b/content/SoftwareEngineering/ProgramingLang/python/Selenium.md similarity index 100% rename from content/ProgramingLang/python/Selenium.md rename to content/SoftwareEngineering/ProgramingLang/python/Selenium.md diff --git a/content/examples in golang.md b/content/SoftwareEngineering/examples in golang.md similarity index 100% rename from content/examples in golang.md rename to content/SoftwareEngineering/examples in golang.md diff --git a/content/hexagonal architecture.md b/content/SoftwareEngineering/hexagonal architecture.md similarity index 100% rename from content/hexagonal architecture.md rename to content/SoftwareEngineering/hexagonal architecture.md diff --git a/content/Untitled 1.canvas b/content/Untitled 1.canvas deleted file mode 100644 index 9e26dfeeb..000000000 --- a/content/Untitled 1.canvas +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/content/Untitled 2.canvas b/content/Untitled 2.canvas deleted file mode 100644 index 9e26dfeeb..000000000 --- a/content/Untitled 2.canvas +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/content/Untitled.canvas b/content/Untitled.canvas deleted file mode 100644 index 9e26dfeeb..000000000 --- a/content/Untitled.canvas +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/content/Pasted image 20240929121007.png b/content/imgs/Pasted image 20240929121007.png similarity index 100% rename from content/Pasted image 20240929121007.png rename to content/imgs/Pasted image 20240929121007.png diff --git a/content/Pasted image 20240929121912.png b/content/imgs/Pasted image 20240929121912.png similarity index 100% rename from content/Pasted image 20240929121912.png rename to content/imgs/Pasted image 20240929121912.png diff --git a/content/Pasted image 20240929122545.png b/content/imgs/Pasted image 20240929122545.png similarity index 100% rename from content/Pasted image 20240929122545.png rename to content/imgs/Pasted image 20240929122545.png diff --git a/content/Screenshot 2024-11-22 at 13.07.15.png b/content/imgs/Screenshot 2024-11-22 at 13.07.15.png similarity index 100% rename from content/Screenshot 2024-11-22 at 13.07.15.png rename to content/imgs/Screenshot 2024-11-22 at 13.07.15.png diff --git a/content/Screenshot 2024-11-22 at 13.20.08.png b/content/imgs/Screenshot 2024-11-22 at 13.20.08.png similarity index 100% rename from content/Screenshot 2024-11-22 at 13.20.08.png rename to content/imgs/Screenshot 2024-11-22 at 13.20.08.png diff --git a/content/Screenshot 2024-11-22 at 13.23.21.png b/content/imgs/Screenshot 2024-11-22 at 13.23.21.png similarity index 100% rename from content/Screenshot 2024-11-22 at 13.23.21.png rename to content/imgs/Screenshot 2024-11-22 at 13.23.21.png diff --git a/content/Screenshot 2024-11-22 at 14.30.18.png b/content/imgs/Screenshot 2024-11-22 at 14.30.18.png similarity index 100% rename from content/Screenshot 2024-11-22 at 14.30.18.png rename to content/imgs/Screenshot 2024-11-22 at 14.30.18.png