diff --git a/AI/assets/build-a-system-prompt-0.png b/AI/assets/build-a-system-prompt-0.png deleted file mode 100644 index 6b9c68dea..000000000 Binary files a/AI/assets/build-a-system-prompt-0.png and /dev/null differ diff --git a/AI/assets/build-a-system-prompt-1.png b/AI/assets/build-a-system-prompt-1.png deleted file mode 100644 index 391bd348c..000000000 Binary files a/AI/assets/build-a-system-prompt-1.png and /dev/null differ diff --git a/AI/assets/build-a-system-prompt-2.gif b/AI/assets/build-a-system-prompt-2.gif deleted file mode 100644 index 71e665487..000000000 Binary files a/AI/assets/build-a-system-prompt-2.gif and /dev/null differ diff --git a/AI/assets/build-a-system-prompt-3.gif b/AI/assets/build-a-system-prompt-3.gif deleted file mode 100644 index 4e1cada33..000000000 Binary files a/AI/assets/build-a-system-prompt-3.gif and /dev/null differ diff --git a/AI/assets/coding-agent-team-0.png b/AI/assets/coding-agent-team-0.png deleted file mode 100644 index a5ae503ce..000000000 Binary files a/AI/assets/coding-agent-team-0.png and /dev/null differ diff --git a/AI/assets/coding-agent-team-1.png b/AI/assets/coding-agent-team-1.png deleted file mode 100644 index 46148f89f..000000000 Binary files a/AI/assets/coding-agent-team-1.png and /dev/null differ diff --git a/AI/assets/coding-agent-team-2.png b/AI/assets/coding-agent-team-2.png deleted file mode 100644 index 38a911fc4..000000000 Binary files a/AI/assets/coding-agent-team-2.png and /dev/null differ diff --git a/_scripts/sh.js b/_scripts/sh.js deleted file mode 100644 index 4e0991f63..000000000 --- a/_scripts/sh.js +++ /dev/null @@ -1,7 +0,0 @@ -async function sh(cmd) { - const { promisify } = require('util'); - const exec = promisify(require('child_process').exec); - const result = await exec(cmd); - return {out: result.stdout.trim(), err: result.stderr.trim()}; -} -module.exports = sh; diff --git a/assets/A perfect binary search tree.png b/assets/A perfect binary search tree.png deleted file mode 100644 index 263ae1557..000000000 Binary files a/assets/A perfect binary search tree.png and /dev/null differ diff --git a/assets/DB_image_1_cropped.png b/assets/DB_image_1_cropped.png deleted file mode 100644 index b558ada32..000000000 Binary files a/assets/DB_image_1_cropped.png and /dev/null differ diff --git a/assets/Draft-choosing.webp b/assets/Draft-choosing.webp deleted file mode 100644 index 227175d1b..000000000 Binary files a/assets/Draft-choosing.webp and /dev/null differ diff --git a/assets/From_2PC_To_Saga.png b/assets/From_2PC_To_Saga.png deleted file mode 100644 index 210f88487..000000000 Binary files a/assets/From_2PC_To_Saga.png and /dev/null differ diff --git a/assets/How to push content on note.d.foundation-20240304213547278.webp b/assets/How to push content on note.d.foundation-20240304213547278.webp deleted file mode 100644 index 3645ae41f..000000000 Binary files a/assets/How to push content on note.d.foundation-20240304213547278.webp and /dev/null differ diff --git a/assets/How to push content on note.d.foundation-20240304213659347.webp b/assets/How to push content on note.d.foundation-20240304213659347.webp deleted file mode 100644 index 4bef95658..000000000 Binary files a/assets/How to push content on note.d.foundation-20240304213659347.webp and /dev/null differ diff --git a/assets/Pasted image 20210810210730.png b/assets/Pasted image 20210810210730.png deleted file mode 100644 index 1985d1c6f..000000000 Binary files a/assets/Pasted image 20210810210730.png and /dev/null differ diff --git a/assets/Pasted image 20210823171027.png b/assets/Pasted image 20210823171027.png deleted file mode 100644 index 52c7534e0..000000000 Binary files a/assets/Pasted image 20210823171027.png and /dev/null differ diff --git a/assets/Pasted image 20220720173320.png b/assets/Pasted image 20220720173320.png deleted file mode 100644 index 2689d156d..000000000 Binary files a/assets/Pasted image 20220720173320.png and /dev/null differ diff --git a/assets/Pattern_Throttling.png b/assets/Pattern_Throttling.png deleted file mode 100644 index f7186a2ec..000000000 Binary files a/assets/Pattern_Throttling.png and /dev/null differ diff --git a/assets/_index__base-20231130183110925.webp b/assets/_index__base-20231130183110925.webp deleted file mode 100644 index adbfd9a59..000000000 Binary files a/assets/_index__base-20231130183110925.webp and /dev/null differ diff --git a/assets/agile-how-to-create-clickup-tickets_0a0e01fd89063d645b741fc4020c6a5b_md5.webp b/assets/agile-how-to-create-clickup-tickets_0a0e01fd89063d645b741fc4020c6a5b_md5.webp deleted file mode 100644 index 1ab5febf8..000000000 Binary files a/assets/agile-how-to-create-clickup-tickets_0a0e01fd89063d645b741fc4020c6a5b_md5.webp and /dev/null differ diff --git a/assets/agile-using-clickup-as-agile-management-tool_3c9972095d36b20c00564bbedce679cf_md5.webp b/assets/agile-using-clickup-as-agile-management-tool_3c9972095d36b20c00564bbedce679cf_md5.webp deleted file mode 100644 index 0f923f4a0..000000000 Binary files a/assets/agile-using-clickup-as-agile-management-tool_3c9972095d36b20c00564bbedce679cf_md5.webp and /dev/null differ diff --git a/assets/agile-using-clickup-as-agile-management-tool_d8e39beb699cc7ddaee213574dc787d7_md5.webp b/assets/agile-using-clickup-as-agile-management-tool_d8e39beb699cc7ddaee213574dc787d7_md5.webp deleted file mode 100644 index 89a38dafc..000000000 Binary files a/assets/agile-using-clickup-as-agile-management-tool_d8e39beb699cc7ddaee213574dc787d7_md5.webp and /dev/null differ diff --git a/assets/agile-using-clickup-as-agile-management-tool_e0d2a879c286008152df56ddf41b28a0_md5.webp b/assets/agile-using-clickup-as-agile-management-tool_e0d2a879c286008152df56ddf41b28a0_md5.webp deleted file mode 100644 index eb69e6f1a..000000000 Binary files a/assets/agile-using-clickup-as-agile-management-tool_e0d2a879c286008152df56ddf41b28a0_md5.webp and /dev/null differ diff --git a/assets/august-forward-engineering_(1).mp4 b/assets/august-forward-engineering_(1).mp4 deleted file mode 100644 index d58ef2e23..000000000 Binary files a/assets/august-forward-engineering_(1).mp4 and /dev/null differ diff --git a/assets/background-tool.png b/assets/background-tool.png deleted file mode 100644 index e9dc7a287..000000000 Binary files a/assets/background-tool.png and /dev/null differ diff --git a/assets/brainery-discord-gomock.webp b/assets/brainery-discord-gomock.webp deleted file mode 100644 index 81cbe1c4e..000000000 Binary files a/assets/brainery-discord-gomock.webp and /dev/null differ diff --git a/assets/brainery-tags-august.webp b/assets/brainery-tags-august.webp deleted file mode 100644 index 04f2f8d0c..000000000 Binary files a/assets/brainery-tags-august.webp and /dev/null differ diff --git a/assets/clean-shot.png b/assets/clean-shot.png deleted file mode 100644 index fa2ccf661..000000000 Binary files a/assets/clean-shot.png and /dev/null differ diff --git a/assets/component-atomic-design.png b/assets/component-atomic-design.png deleted file mode 100644 index 8a3d2af5f..000000000 Binary files a/assets/component-atomic-design.png and /dev/null differ diff --git a/assets/console.dir()_example.jpg b/assets/console.dir()_example.jpg deleted file mode 100644 index 353c9bb30..000000000 Binary files a/assets/console.dir()_example.jpg and /dev/null differ diff --git a/assets/cost-of-react-native_076bec90671c2f71c0fdb93849498a01_md5.webp b/assets/cost-of-react-native_076bec90671c2f71c0fdb93849498a01_md5.webp deleted file mode 100644 index be1710abb..000000000 Binary files a/assets/cost-of-react-native_076bec90671c2f71c0fdb93849498a01_md5.webp and /dev/null differ diff --git a/assets/cost-of-react-native_69b49ec589ff19151408b8959ace27bd_md5.webp b/assets/cost-of-react-native_69b49ec589ff19151408b8959ace27bd_md5.webp deleted file mode 100644 index b4f6851be..000000000 Binary files a/assets/cost-of-react-native_69b49ec589ff19151408b8959ace27bd_md5.webp and /dev/null differ diff --git a/assets/cost-of-react-native_cb4a01ca2ab93832ae3ffec401441e07_md5.webp b/assets/cost-of-react-native_cb4a01ca2ab93832ae3ffec401441e07_md5.webp deleted file mode 100644 index f405fd9a9..000000000 Binary files a/assets/cost-of-react-native_cb4a01ca2ab93832ae3ffec401441e07_md5.webp and /dev/null differ diff --git a/assets/distributed-access-platform.webp b/assets/distributed-access-platform.webp deleted file mode 100644 index 8ac51e4ac..000000000 Binary files a/assets/distributed-access-platform.webp and /dev/null differ diff --git a/assets/droppii-dwarves-case-study-20240103153144822.webp b/assets/droppii-dwarves-case-study-20240103153144822.webp deleted file mode 100644 index e14e54290..000000000 Binary files a/assets/droppii-dwarves-case-study-20240103153144822.webp and /dev/null differ diff --git a/assets/droppii-dwarves-case-study-20240103153210322.webp b/assets/droppii-dwarves-case-study-20240103153210322.webp deleted file mode 100644 index 557206f43..000000000 Binary files a/assets/droppii-dwarves-case-study-20240103153210322.webp and /dev/null differ diff --git a/assets/droppii-dwarves-case-study-20240103153226614.webp b/assets/droppii-dwarves-case-study-20240103153226614.webp deleted file mode 100644 index 388169b29..000000000 Binary files a/assets/droppii-dwarves-case-study-20240103153226614.webp and /dev/null differ diff --git a/assets/dwarves-community-20231215165531369.webp b/assets/dwarves-community-20231215165531369.webp deleted file mode 100644 index 8f5eadd61..000000000 Binary files a/assets/dwarves-community-20231215165531369.webp and /dev/null differ diff --git a/assets/dwarves-community-20240304213917412.webp b/assets/dwarves-community-20240304213917412.webp deleted file mode 100644 index b8761ba7b..000000000 Binary files a/assets/dwarves-community-20240304213917412.webp and /dev/null differ diff --git a/assets/dwarves-rewind.webp b/assets/dwarves-rewind.webp deleted file mode 100644 index 8502312b5..000000000 Binary files a/assets/dwarves-rewind.webp and /dev/null differ diff --git a/assets/erd-ai-generated.webp b/assets/erd-ai-generated.webp deleted file mode 100644 index 0c5d56d6f..000000000 Binary files a/assets/erd-ai-generated.webp and /dev/null differ diff --git a/assets/evo-db.pdf b/assets/evo-db.pdf deleted file mode 100644 index e7d85aa81..000000000 Binary files a/assets/evo-db.pdf and /dev/null differ diff --git a/assets/example-build-vercel.png b/assets/example-build-vercel.png deleted file mode 100644 index ae109b061..000000000 Binary files a/assets/example-build-vercel.png and /dev/null differ diff --git a/assets/finite-state-machine_93ba1268646d3675466aa8887079d580_md5.webp b/assets/finite-state-machine_93ba1268646d3675466aa8887079d580_md5.webp deleted file mode 100644 index 79bc5535f..000000000 Binary files a/assets/finite-state-machine_93ba1268646d3675466aa8887079d580_md5.webp and /dev/null differ diff --git a/assets/finite-state-machine_b5d7d35ae8b3b4cbd6c0c9d8589d4dae_md5.webp b/assets/finite-state-machine_b5d7d35ae8b3b4cbd6c0c9d8589d4dae_md5.webp deleted file mode 100644 index 44c3e5291..000000000 Binary files a/assets/finite-state-machine_b5d7d35ae8b3b4cbd6c0c9d8589d4dae_md5.webp and /dev/null differ diff --git a/assets/finite-state-machine_db5600646453d67b0df29dfa3c9dc5b0_md5.webp b/assets/finite-state-machine_db5600646453d67b0df29dfa3c9dc5b0_md5.webp deleted file mode 100644 index ce15cd9e3..000000000 Binary files a/assets/finite-state-machine_db5600646453d67b0df29dfa3c9dc5b0_md5.webp and /dev/null differ diff --git a/assets/fwd-engineering-columns.png b/assets/fwd-engineering-columns.png deleted file mode 100644 index c43b8d6eb..000000000 Binary files a/assets/fwd-engineering-columns.png and /dev/null differ diff --git a/assets/fwd-engineering-flow.png b/assets/fwd-engineering-flow.png deleted file mode 100644 index 1427e5a99..000000000 Binary files a/assets/fwd-engineering-flow.png and /dev/null differ diff --git a/assets/fwd-engineering-subscribe.png b/assets/fwd-engineering-subscribe.png deleted file mode 100644 index fa3625639..000000000 Binary files a/assets/fwd-engineering-subscribe.png and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg-2.webp b/assets/how-to-earn-reward-from-staking-dfg-2.webp deleted file mode 100644 index 1b2d7b8da..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg-2.webp and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg-3.png.png b/assets/how-to-earn-reward-from-staking-dfg-3.png.png deleted file mode 100644 index 8a41387cb..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg-3.png.png and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg-4.png.png b/assets/how-to-earn-reward-from-staking-dfg-4.png.png deleted file mode 100644 index bedb806ff..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg-4.png.png and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg.webp b/assets/how-to-earn-reward-from-staking-dfg.webp deleted file mode 100644 index 0504b6a59..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg.webp and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-35-22-2x.webp b/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-35-22-2x.webp deleted file mode 100644 index 6a897e354..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-35-22-2x.webp and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-37-11-2x.webp b/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-37-11-2x.webp deleted file mode 100644 index 33fd57058..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-37-11-2x.webp and /dev/null differ diff --git a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-39-16-2x.webp b/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-39-16-2x.webp deleted file mode 100644 index 5d4893e41..000000000 Binary files a/assets/how-to-earn-reward-from-staking-dfg_clean-shot-2024-05-13-at-16-39-16-2x.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-1 2.webp b/assets/how-to-withdraw-icy-1 2.webp deleted file mode 100644 index 22824cf18..000000000 Binary files a/assets/how-to-withdraw-icy-1 2.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-2 2.webp b/assets/how-to-withdraw-icy-2 2.webp deleted file mode 100644 index e3df317a0..000000000 Binary files a/assets/how-to-withdraw-icy-2 2.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313132819285.webp b/assets/how-to-withdraw-icy-20240313132819285.webp deleted file mode 100644 index a8dcecc34..000000000 Binary files a/assets/how-to-withdraw-icy-20240313132819285.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313132855655.webp b/assets/how-to-withdraw-icy-20240313132855655.webp deleted file mode 100644 index e36cd9546..000000000 Binary files a/assets/how-to-withdraw-icy-20240313132855655.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313143503939.webp b/assets/how-to-withdraw-icy-20240313143503939.webp deleted file mode 100644 index 08f316873..000000000 Binary files a/assets/how-to-withdraw-icy-20240313143503939.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313143524973.webp b/assets/how-to-withdraw-icy-20240313143524973.webp deleted file mode 100644 index 9b8566102..000000000 Binary files a/assets/how-to-withdraw-icy-20240313143524973.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313143710295.webp b/assets/how-to-withdraw-icy-20240313143710295.webp deleted file mode 100644 index cc2457cd5..000000000 Binary files a/assets/how-to-withdraw-icy-20240313143710295.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313143732060.webp b/assets/how-to-withdraw-icy-20240313143732060.webp deleted file mode 100644 index aac28ac73..000000000 Binary files a/assets/how-to-withdraw-icy-20240313143732060.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313143849627.webp b/assets/how-to-withdraw-icy-20240313143849627.webp deleted file mode 100644 index bd19ff3f7..000000000 Binary files a/assets/how-to-withdraw-icy-20240313143849627.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144057277.webp b/assets/how-to-withdraw-icy-20240313144057277.webp deleted file mode 100644 index 97b4e078d..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144057277.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144103209.webp b/assets/how-to-withdraw-icy-20240313144103209.webp deleted file mode 100644 index dbd9b1bde..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144103209.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144120524.webp b/assets/how-to-withdraw-icy-20240313144120524.webp deleted file mode 100644 index 6ebcb6518..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144120524.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144142293.webp b/assets/how-to-withdraw-icy-20240313144142293.webp deleted file mode 100644 index 9a1f1ad3c..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144142293.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144212466.webp b/assets/how-to-withdraw-icy-20240313144212466.webp deleted file mode 100644 index 7e6007409..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144212466.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144220528.webp b/assets/how-to-withdraw-icy-20240313144220528.webp deleted file mode 100644 index bc29bb8f8..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144220528.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144251314.webp b/assets/how-to-withdraw-icy-20240313144251314.webp deleted file mode 100644 index 7427d938b..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144251314.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144301936.webp b/assets/how-to-withdraw-icy-20240313144301936.webp deleted file mode 100644 index eed3ed993..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144301936.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144410171.webp b/assets/how-to-withdraw-icy-20240313144410171.webp deleted file mode 100644 index 4202ba912..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144410171.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144425125.webp b/assets/how-to-withdraw-icy-20240313144425125.webp deleted file mode 100644 index abf1c70b4..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144425125.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144450367.webp b/assets/how-to-withdraw-icy-20240313144450367.webp deleted file mode 100644 index 3d2d260d3..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144450367.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144456227.webp b/assets/how-to-withdraw-icy-20240313144456227.webp deleted file mode 100644 index 88887b0ef..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144456227.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144633452.webp b/assets/how-to-withdraw-icy-20240313144633452.webp deleted file mode 100644 index 22c3fcadd..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144633452.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144637789.webp b/assets/how-to-withdraw-icy-20240313144637789.webp deleted file mode 100644 index 867c28dd7..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144637789.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144652671.webp b/assets/how-to-withdraw-icy-20240313144652671.webp deleted file mode 100644 index 6faf38bb2..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144652671.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144732450.webp b/assets/how-to-withdraw-icy-20240313144732450.webp deleted file mode 100644 index 9692fac9e..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144732450.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144815006.webp b/assets/how-to-withdraw-icy-20240313144815006.webp deleted file mode 100644 index 8c893d52b..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144815006.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144835406.webp b/assets/how-to-withdraw-icy-20240313144835406.webp deleted file mode 100644 index 3305c4112..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144835406.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144845614.webp b/assets/how-to-withdraw-icy-20240313144845614.webp deleted file mode 100644 index bc797b9c3..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144845614.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144859473.webp b/assets/how-to-withdraw-icy-20240313144859473.webp deleted file mode 100644 index 89cad7a0c..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144859473.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144911243.webp b/assets/how-to-withdraw-icy-20240313144911243.webp deleted file mode 100644 index 9f45041df..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144911243.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144922569.webp b/assets/how-to-withdraw-icy-20240313144922569.webp deleted file mode 100644 index fe62934ed..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144922569.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144951021.webp b/assets/how-to-withdraw-icy-20240313144951021.webp deleted file mode 100644 index 17bc39930..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144951021.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313144956988.webp b/assets/how-to-withdraw-icy-20240313144956988.webp deleted file mode 100644 index ad32ece2c..000000000 Binary files a/assets/how-to-withdraw-icy-20240313144956988.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313145039356.webp b/assets/how-to-withdraw-icy-20240313145039356.webp deleted file mode 100644 index 3f5b27528..000000000 Binary files a/assets/how-to-withdraw-icy-20240313145039356.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313145048322.webp b/assets/how-to-withdraw-icy-20240313145048322.webp deleted file mode 100644 index d2c610631..000000000 Binary files a/assets/how-to-withdraw-icy-20240313145048322.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy-20240313145055489.webp b/assets/how-to-withdraw-icy-20240313145055489.webp deleted file mode 100644 index 2236d42e6..000000000 Binary files a/assets/how-to-withdraw-icy-20240313145055489.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy_install-1.webp b/assets/how-to-withdraw-icy_install-1.webp deleted file mode 100644 index 65e377f1b..000000000 Binary files a/assets/how-to-withdraw-icy_install-1.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy_install-4.webp b/assets/how-to-withdraw-icy_install-4.webp deleted file mode 100644 index 500e9ccc0..000000000 Binary files a/assets/how-to-withdraw-icy_install-4.webp and /dev/null differ diff --git a/assets/how-to-withdraw-icy_withdraw-3.webp b/assets/how-to-withdraw-icy_withdraw-3.webp deleted file mode 100644 index 813c25f45..000000000 Binary files a/assets/how-to-withdraw-icy_withdraw-3.webp and /dev/null differ diff --git a/assets/icy/salary-advance-amount.jpg b/assets/icy/salary-advance-amount.jpg deleted file mode 100644 index 47d71ef68..000000000 Binary files a/assets/icy/salary-advance-amount.jpg and /dev/null differ diff --git a/assets/labs---new-member-onboarding-20231121174522981.webp b/assets/labs---new-member-onboarding-20231121174522981.webp deleted file mode 100644 index 90819f165..000000000 Binary files a/assets/labs---new-member-onboarding-20231121174522981.webp and /dev/null differ diff --git a/assets/labs---new-member-onboarding-20231205185801920.webp b/assets/labs---new-member-onboarding-20231205185801920.webp deleted file mode 100644 index cb268303b..000000000 Binary files a/assets/labs---new-member-onboarding-20231205185801920.webp and /dev/null differ diff --git a/assets/labs---topic-proposal-&-progress-tracking_basecamp-hill-chart.webp b/assets/labs---topic-proposal-&-progress-tracking_basecamp-hill-chart.webp deleted file mode 100644 index d906c88c1..000000000 Binary files a/assets/labs---topic-proposal-&-progress-tracking_basecamp-hill-chart.webp and /dev/null differ diff --git a/assets/labs---topic-proposal-&-progress-tracking_basecamp-todo.webp b/assets/labs---topic-proposal-&-progress-tracking_basecamp-todo.webp deleted file mode 100644 index cd1167899..000000000 Binary files a/assets/labs---topic-proposal-&-progress-tracking_basecamp-todo.webp and /dev/null differ diff --git a/assets/labs-roadmap-nov-23-update-20231127143515779.webp b/assets/labs-roadmap-nov-23-update-20231127143515779.webp deleted file mode 100644 index 3a2ccc6f5..000000000 Binary files a/assets/labs-roadmap-nov-23-update-20231127143515779.webp and /dev/null differ diff --git a/assets/labs-x-consulting-workflow-20231121174237957.webp b/assets/labs-x-consulting-workflow-20231121174237957.webp deleted file mode 100644 index c54b89ccb..000000000 Binary files a/assets/labs-x-consulting-workflow-20231121174237957.webp and /dev/null differ diff --git a/assets/labs-x-consulting-workflow_labs-topic-timeline-20231121174237957.webp b/assets/labs-x-consulting-workflow_labs-topic-timeline-20231121174237957.webp deleted file mode 100644 index 3510be142..000000000 Binary files a/assets/labs-x-consulting-workflow_labs-topic-timeline-20231121174237957.webp and /dev/null differ diff --git a/assets/llm-story-map.webp b/assets/llm-story-map.webp deleted file mode 100644 index fbc3b4e40..000000000 Binary files a/assets/llm-story-map.webp and /dev/null differ diff --git a/assets/product-engineer-twitter.webp b/assets/product-engineer-twitter.webp deleted file mode 100644 index a283c1614..000000000 Binary files a/assets/product-engineer-twitter.webp and /dev/null differ diff --git a/assets/pubsub.png b/assets/pubsub.png deleted file mode 100644 index 143482e18..000000000 Binary files a/assets/pubsub.png and /dev/null differ diff --git a/assets/push-discord-to-github-bounty_untitled-9.webp b/assets/push-discord-to-github-bounty_untitled-9.webp deleted file mode 100644 index e1765c287..000000000 Binary files a/assets/push-discord-to-github-bounty_untitled-9.webp and /dev/null differ diff --git a/assets/rag-ai.webp b/assets/rag-ai.webp deleted file mode 100644 index 1402b532b..000000000 Binary files a/assets/rag-ai.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_00906c2cfe7b99ad8994125f2dad50e8_md5.webp b/assets/react-native-new-architecture_00906c2cfe7b99ad8994125f2dad50e8_md5.webp deleted file mode 100644 index d334ba400..000000000 Binary files a/assets/react-native-new-architecture_00906c2cfe7b99ad8994125f2dad50e8_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_044af91a93a00b09540b03b991e4e32a_md5.webp b/assets/react-native-new-architecture_044af91a93a00b09540b03b991e4e32a_md5.webp deleted file mode 100644 index 48918733f..000000000 Binary files a/assets/react-native-new-architecture_044af91a93a00b09540b03b991e4e32a_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_08f9602cedbc58744cf72fc3580bf068_md5.webp b/assets/react-native-new-architecture_08f9602cedbc58744cf72fc3580bf068_md5.webp deleted file mode 100644 index ff5bb957a..000000000 Binary files a/assets/react-native-new-architecture_08f9602cedbc58744cf72fc3580bf068_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_1066bb62cbc4d9beda7e03d43006c669_md5.webp b/assets/react-native-new-architecture_1066bb62cbc4d9beda7e03d43006c669_md5.webp deleted file mode 100644 index 73647b421..000000000 Binary files a/assets/react-native-new-architecture_1066bb62cbc4d9beda7e03d43006c669_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_193938237761968661a5f11f23670f3d_md5.webp b/assets/react-native-new-architecture_193938237761968661a5f11f23670f3d_md5.webp deleted file mode 100644 index bd573f4c4..000000000 Binary files a/assets/react-native-new-architecture_193938237761968661a5f11f23670f3d_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_be56fa7f4ae19c9284e37e61706d1133_md5.webp b/assets/react-native-new-architecture_be56fa7f4ae19c9284e37e61706d1133_md5.webp deleted file mode 100644 index 785f8fad5..000000000 Binary files a/assets/react-native-new-architecture_be56fa7f4ae19c9284e37e61706d1133_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_ca02c7ac30b22ad423bf3285f8cdaa16_md5.webp b/assets/react-native-new-architecture_ca02c7ac30b22ad423bf3285f8cdaa16_md5.webp deleted file mode 100644 index 96b8955fc..000000000 Binary files a/assets/react-native-new-architecture_ca02c7ac30b22ad423bf3285f8cdaa16_md5.webp and /dev/null differ diff --git a/assets/react-native-new-architecture_fb23cab260bd0ec3a95ac7d6a69c461a_md5.webp b/assets/react-native-new-architecture_fb23cab260bd0ec3a95ac7d6a69c461a_md5.webp deleted file mode 100644 index 2d76c38a7..000000000 Binary files a/assets/react-native-new-architecture_fb23cab260bd0ec3a95ac7d6a69c461a_md5.webp and /dev/null differ diff --git a/assets/setup-react-project-with-webpack-and-babel_0eb458d1be6d0f37210105e714a1a304_md5.webp b/assets/setup-react-project-with-webpack-and-babel_0eb458d1be6d0f37210105e714a1a304_md5.webp deleted file mode 100644 index c7166f0ce..000000000 Binary files a/assets/setup-react-project-with-webpack-and-babel_0eb458d1be6d0f37210105e714a1a304_md5.webp and /dev/null differ diff --git a/assets/setup-react-project-with-webpack-and-babel_1390995c704e49aab8915f9a87fcd7ce_md5.webp b/assets/setup-react-project-with-webpack-and-babel_1390995c704e49aab8915f9a87fcd7ce_md5.webp deleted file mode 100644 index 4c4835317..000000000 Binary files a/assets/setup-react-project-with-webpack-and-babel_1390995c704e49aab8915f9a87fcd7ce_md5.webp and /dev/null differ diff --git a/assets/strangler.png b/assets/strangler.png deleted file mode 100644 index b39b55a1f..000000000 Binary files a/assets/strangler.png and /dev/null differ diff --git a/assets/update-highest-and-lowest-symbol-prices-in-real-time.png b/assets/update-highest-and-lowest-symbol-prices-in-real-time.png deleted file mode 100644 index 9a4fd45f8..000000000 Binary files a/assets/update-highest-and-lowest-symbol-prices-in-real-time.png and /dev/null differ diff --git a/assets/utilizing-cached-table-for-binance-kline-api-data-processing.png b/assets/utilizing-cached-table-for-binance-kline-api-data-processing.png deleted file mode 100644 index 65e2ddbb2..000000000 Binary files a/assets/utilizing-cached-table-for-binance-kline-api-data-processing.png and /dev/null differ diff --git a/breakdown/assets/ax-framework-cheatsheet.png b/breakdown/assets/ax-framework-cheatsheet.png deleted file mode 100644 index 6b81d495a..000000000 Binary files a/breakdown/assets/ax-framework-cheatsheet.png and /dev/null differ diff --git a/breakdown/assets/ax_optimize.png b/breakdown/assets/ax_optimize.png deleted file mode 100644 index 7c0aeb969..000000000 Binary files a/breakdown/assets/ax_optimize.png and /dev/null differ diff --git a/breakdown/assets/ax_signature.png b/breakdown/assets/ax_signature.png deleted file mode 100644 index 44623e5f1..000000000 Binary files a/breakdown/assets/ax_signature.png and /dev/null differ diff --git a/breakdown/assets/cap-instant-mode.gif b/breakdown/assets/cap-instant-mode.gif deleted file mode 100644 index 1ae11e1ed..000000000 Binary files a/breakdown/assets/cap-instant-mode.gif and /dev/null differ diff --git a/breakdown/assets/cap-muxing.png b/breakdown/assets/cap-muxing.png deleted file mode 100644 index 90a45d739..000000000 Binary files a/breakdown/assets/cap-muxing.png and /dev/null differ diff --git a/breakdown/assets/cline-cheatsheet.png b/breakdown/assets/cline-cheatsheet.png deleted file mode 100644 index 699576efc..000000000 Binary files a/breakdown/assets/cline-cheatsheet.png and /dev/null differ diff --git a/breakdown/assets/cline-container-architect.png b/breakdown/assets/cline-container-architect.png deleted file mode 100644 index 60872c51c..000000000 Binary files a/breakdown/assets/cline-container-architect.png and /dev/null differ diff --git a/breakdown/assets/cline-core-components.png b/breakdown/assets/cline-core-components.png deleted file mode 100644 index c3840454c..000000000 Binary files a/breakdown/assets/cline-core-components.png and /dev/null differ diff --git a/breakdown/assets/cline-illu.gif b/breakdown/assets/cline-illu.gif deleted file mode 100644 index 82953fb7a..000000000 Binary files a/breakdown/assets/cline-illu.gif and /dev/null differ diff --git a/breakdown/assets/cline-system-flows.png b/breakdown/assets/cline-system-flows.png deleted file mode 100644 index 8f86bc32f..000000000 Binary files a/breakdown/assets/cline-system-flows.png and /dev/null differ diff --git a/breakdown/assets/context7-refresh-library.png b/breakdown/assets/context7-refresh-library.png deleted file mode 100644 index 601165eea..000000000 Binary files a/breakdown/assets/context7-refresh-library.png and /dev/null differ diff --git a/breakdown/assets/context7-token-limit.gif b/breakdown/assets/context7-token-limit.gif deleted file mode 100644 index cd41d3bb3..000000000 Binary files a/breakdown/assets/context7-token-limit.gif and /dev/null differ diff --git a/breakdown/assets/crawl4ai-cheatsheet.png b/breakdown/assets/crawl4ai-cheatsheet.png deleted file mode 100644 index 4ea7655bf..000000000 Binary files a/breakdown/assets/crawl4ai-cheatsheet.png and /dev/null differ diff --git a/breakdown/assets/dify-architecture.webp b/breakdown/assets/dify-architecture.webp deleted file mode 100644 index 87ea880af..000000000 Binary files a/breakdown/assets/dify-architecture.webp and /dev/null differ diff --git a/breakdown/assets/dify-load-distribution.webp b/breakdown/assets/dify-load-distribution.webp deleted file mode 100644 index fba6bd840..000000000 Binary files a/breakdown/assets/dify-load-distribution.webp and /dev/null differ diff --git a/breakdown/assets/dify-model-runtime-layer.webp b/breakdown/assets/dify-model-runtime-layer.webp deleted file mode 100644 index 402a0864b..000000000 Binary files a/breakdown/assets/dify-model-runtime-layer.webp and /dev/null differ diff --git a/breakdown/assets/dify-plugin-ecosystem.webp b/breakdown/assets/dify-plugin-ecosystem.webp deleted file mode 100644 index 34904429e..000000000 Binary files a/breakdown/assets/dify-plugin-ecosystem.webp and /dev/null differ diff --git a/breakdown/assets/dify-workflow-execution-engine.webp b/breakdown/assets/dify-workflow-execution-engine.webp deleted file mode 100644 index bfe7da1a6..000000000 Binary files a/breakdown/assets/dify-workflow-execution-engine.webp and /dev/null differ diff --git a/breakdown/assets/dify.gif b/breakdown/assets/dify.gif deleted file mode 100644 index 1e30f5a15..000000000 Binary files a/breakdown/assets/dify.gif and /dev/null differ diff --git a/breakdown/assets/e2b-illustration-01.png b/breakdown/assets/e2b-illustration-01.png deleted file mode 100644 index 5affe846f..000000000 Binary files a/breakdown/assets/e2b-illustration-01.png and /dev/null differ diff --git a/breakdown/assets/e2b-illustration-02.png b/breakdown/assets/e2b-illustration-02.png deleted file mode 100644 index 6822ee26d..000000000 Binary files a/breakdown/assets/e2b-illustration-02.png and /dev/null differ diff --git a/breakdown/assets/maybe-illu.gif b/breakdown/assets/maybe-illu.gif deleted file mode 100644 index 0068bb2b3..000000000 Binary files a/breakdown/assets/maybe-illu.gif and /dev/null differ diff --git a/breakdown/assets/maybe-multi-currency.png b/breakdown/assets/maybe-multi-currency.png deleted file mode 100644 index d1bf9bff5..000000000 Binary files a/breakdown/assets/maybe-multi-currency.png and /dev/null differ diff --git a/breakdown/assets/mem0-benchmarck-latency.png b/breakdown/assets/mem0-benchmarck-latency.png deleted file mode 100644 index 9f100a21b..000000000 Binary files a/breakdown/assets/mem0-benchmarck-latency.png and /dev/null differ diff --git a/breakdown/assets/mem0-graph-architecture.png b/breakdown/assets/mem0-graph-architecture.png deleted file mode 100644 index 7d8b11729..000000000 Binary files a/breakdown/assets/mem0-graph-architecture.png and /dev/null differ diff --git a/breakdown/assets/mem0-vector-architecture.png b/breakdown/assets/mem0-vector-architecture.png deleted file mode 100644 index 077eb9b54..000000000 Binary files a/breakdown/assets/mem0-vector-architecture.png and /dev/null differ diff --git a/breakdown/assets/stagehand.gif b/breakdown/assets/stagehand.gif deleted file mode 100644 index 9b105cb17..000000000 Binary files a/breakdown/assets/stagehand.gif and /dev/null differ diff --git a/breakdown/assets/umami.gif b/breakdown/assets/umami.gif deleted file mode 100644 index ee53d8e43..000000000 Binary files a/breakdown/assets/umami.gif and /dev/null differ diff --git a/breakdown/assets/zen-mcp-cheatsheet.png b/breakdown/assets/zen-mcp-cheatsheet.png deleted file mode 100644 index 2d6a77dfa..000000000 Binary files a/breakdown/assets/zen-mcp-cheatsheet.png and /dev/null differ diff --git a/notes/Acceptance test in Elixir.md b/notes/Acceptance test in Elixir.md deleted file mode 100644 index 734de349a..000000000 --- a/notes/Acceptance test in Elixir.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir - - engineering - - testing -redirect: - - /4VGSXQ ---- - -Acceptance tests help ensure that applications we develop are deployed with minimal bugs. Manual acceptance testing is extremely valuable, but it can also be tedious and time-consuming. At DF, we've applied the automation End2End testing for our development process. In End2End test, we can open the web page, navigate on the web like a real user. In elixir project, we can do the same thing with [Hound](https://github.com/HashNuke/hound) and a WebDriver. - -The WebDriver is an interface to automate browsers. We can use WebDriver implementation to build an automated test suite against a real browser. - -Hound is an Elixir library to write WebDriver. Writing End2End test with Hound is very similar to writing a normal ExUnit test. An advantage to writing automation tests is the same code base. - -### Common web drivers - -https://github.com/HashNuke/hound/wiki/Starting-a-webdriver-server - -[**Selenium**](https://selenium-release.storage.googleapis.com/index.html) - -```shell -$ brew install selenium-server-standalone -``` - -[**ChromeDriver server**](https://chromedriver.chromium.org) - -```shell -$ brew install --cask chromedriver -``` - -Run command: - -```shell -$ chromedriver -``` - -**PhantomJS web driver** - -```shell -$ brew cask install phantomjs -``` - -Run command: - -```shell -$ phantomjs --wd -``` - -### Configure Hound - -Add dependency to `mix.exs`: - -```elixir -{:hound, "~> 1.0"} -``` - -Follow the setup step in https://hexdocs.pm/phoenix_ecto/Phoenix.Ecto.SQL.Sandbox.html: - -- add the plug to `Endpoint` -- update the config/test.exs to run real server when execute test cases -- add the hound config in config/test.exs. We can update the config for other driver and browser config. - -```elixir -config :hound, driver: "chrome_driver", browser: "chrome_headless" -``` - -- Update `test/test_helper.ex`. We must start :hound application before run the unit test. - -```elixir -Application.ensure_all_started(:hound) -``` - -### Make an unit test - -As any unit test we need a module to store the test cases. We implement the `setup` func for Web driver initial `Hound.Helpers` provides the functions to control the browser easier. We can navigate, inspect html element, make the click action via this package. Use `assert` to validate the result - -```elixir -defmodule Acceptance.HomePageTest do - use ExUnit.Case, async: false - use Hound.Helpers - - setup do - Hound.start_session() - :ok - end - - test "the page loads" do - navigate_to("http://localhost:4002") - assert page_title() == "Homepage" - end -end -``` - -### Note - -Before run test we MUST start the web driver first https://hexdocs.pm/hound/Hound.Helpers.Navigation.html diff --git a/notes/Adding aliases and import into mix file under umbrella app.md b/notes/Adding aliases and import into mix file under umbrella app.md deleted file mode 100644 index 8302caee8..000000000 --- a/notes/Adding aliases and import into mix file under umbrella app.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir -redirect: - - /MScKIQ ---- - -Here is the content of my mix file: `.iex.exs` - -```elixir -alias Db.{ Repo, Document } -``` - ---- - -#### Reference - -- https://stackoverflow.com/questions/46842465/how-to-add-aliases-and-imports-into-my-own-mix-under-umbrella-application-in-eli?rq=1 diff --git a/notes/Benefit of Umbrella Application.md b/notes/Benefit of Umbrella Application.md deleted file mode 100644 index 2d37800b1..000000000 --- a/notes/Benefit of Umbrella Application.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir -redirect: - - /fwdKEQ ---- - -An advantage to using the umbrella structure is that the applications are structured as standalone applications. This can help to give clearer boundaries between the applications and to separate application concerns. For example, an application in the umbrella should only call the public apis of another umbrella application. However, this is not enforced and private apis can still be called. - -The individual applications in the umbrella have their own configuration files, tests and separate `mix.exs` files which can be compiled, run and tested as one big application because they are in the umbrella. However, a single application can still be run and tested standalone when working inside the application directory (in `apps/app_name`). - -In my opinion there can also be drawbacks to this. A drawback to the umbrella is that you have `1 + app_count` different config and mix files to manage. Additionally your source files will be nested deeper, e.g. in `apps/app_name/lib/app_name/...` instead of just `lib/app_name`, which makes it harder to navigate the source code. - -In many cases I find that an umbrella app is not necessary for the project I'm working on, because the applications in the umbrella are never going to run standalone anyway. Often I prefer having projects with different namespaces in the `lib/` folder while still treating them as applications with separate responsiblities. - ---- - -#### Reference - -- [Docs for "Dependencies and umbrella projects" at elixir-lang.org](https://elixir-lang.org/getting-started/mix-otp/dependencies-and-umbrella-projects.html) -- [Benefit of umbrella application](https://stackoverflow.com/questions/59958055/what-are-the-benefits-of-umbrella-applications) diff --git a/notes/Blockchain Oracle.md b/notes/Blockchain Oracle.md deleted file mode 100644 index 8dc6b1337..000000000 --- a/notes/Blockchain Oracle.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: null -description: null -date: null -author: Vy Tran Khac -tags: - - blockchain - - engineering -redirect: - - /Y7zXkg ---- - -## The blockchain oracle problem - -Blockchains have a fundamental limitation: they cannot natively communicate with systems of the outside world. This lack of external connectivity, known as "the oracle problem", prevents smart contracts from verifying external events, trigger actions on existing systems, and providing users the full range of functionality. - -## What is a blockchain oracle? - -A blockchain oracle is a third-party service that connects smart contracts with the outside world, primarily to feed information in about the world around, but the reverse is also true. - -![ins and outs of the blockchain ecosystem](https://i.imgur.com/JS9ezGw.png) _Blockchain oracles connect blockchains to inputs and outputs of the real world (Image source: [Chainlink](https://chain.link/))_ - -## Types of blockchain oracles - -### Input oracles - -The most widely recognized type of oracle today is known as an “input oracle”, which fetches data from the real-world(off-chain) and delivers it onto a blockchain network for smart contracts consumption. A good example of this are the Chainlink Price Feeds. - -### Output oracles - -The opposite of input oracles are "output oracles", which allow smart contracts to send commands to off-chain systems to trigger and execute certain actions. - -### Cross-chain oracles - -This type of oracle can read and write information between different blockchains. Cross-chain oracles enable interoperability for moving both data and assets between blockchains. - -### Compute-enabled oracles - -This type of oracle provides decentralized services with secure, off-chain computation that would otherwise be impractical to do on the blockchain due to technical, legal, or financial constraints. - -## Notable blockchain oracles - -- Chainlink (LINK) -- Band Protocol (BAND) -- Teller (TRB) -- Decentralized Information Asset (DIA) -- API3 - -#### Reference - -- [Wikipedia - Blockchain oracle](https://en.wikipedia.org/wiki/Blockchain_oracle#:~:text=A%20blockchain%20oracle%20is%20a,that%20decentralised%20knowledge%20is%20obtained.) -- [https://chain.link/education/blockchain-oracles](https://chain.link/education/blockchain-oracles) -- [https://coin98.net/what-is-blockchain-oracle](https://coin98.net/what-is-blockchain-oracle) diff --git a/notes/CICD.md b/notes/CICD.md deleted file mode 100644 index e13a13dbc..000000000 --- a/notes/CICD.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - development -redirect: - - /MPAZ6w ---- - -### CI - -- Avoid merge conflicts -- Auto merge source code from different locals -- Constantly build & test -- Notify when bug arrives -- Only merge if `no errors` - -### CD - -- After code is compiled & merge -- Code is moved to another env for more test -- Passed these tests, code is ready for production -- CD ensures a safety net for production code - - i.e: once code is shipped to production, it can still be tested and notify bugs - -### The CI/CD pipeline - -- Automatic // infinite loop of testing, notifying and building -- Saves times. prevents developers to deal with merge conflicts and bugs -- Provides constant feedback -- The pipeline keeps traverse between production // source code for bug tracking - ---- - -#### Reference - -- https://kylefarmer85.medium.com/what-is-ci-cd-76d71b82898d diff --git a/notes/CRON Job.md b/notes/CRON Job.md deleted file mode 100644 index 82b655170..000000000 --- a/notes/CRON Job.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - development - - practice -redirect: - - /PGAHBg ---- - -### What - -A system daemon to execute task on designed times. - -_daemon_: run in linux system. works underlying in background - -It's a long-running process to execute command at specifice date/time - -### How it works - -Using crontab file. A configed text file, contains a list of command to schedule tasks. This file can be edit/ adjust using crontab command - -_Crontab Command_: List of action you wish to execute - -### Example - -**Format** - -- Minute -- Hour -- Day -- Month -- Day of the week -- Command - -`30 09 ** 08 * / home/ramesh/update-db` - -Task: Auto update database at 09:30AM on every day of August - ---- - -#### Reference - -- https://ostechnix.com/a-beginners-guide-to-cron-jobs/ -- https://www.marksanborn.net/linux/learning-cron-by-example/ -- https://www.thegeekstuff.com/2009/06/15-practical-crontab-examples/ diff --git a/notes/Circuit breaker Pattern.md b/notes/Circuit breaker Pattern.md deleted file mode 100644 index cddc28487..000000000 --- a/notes/Circuit breaker Pattern.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - cloud-pattern - - engineering - - microservice -redirect: - - /bNuXhA ---- - -### Context - -- In a distributed environment, there can be situations where faults are due to unanticipated events, and that might take much longer to fix. Result the complete failure of a service. In these situations it might be pointless for an application to continually retry an operation that is unlikely to succeed, and instead the application should quickly accept that the operation has failed and handle this failure accordingly. -- Sometime, failed service could have bad effect to other services about: delay, performance and reduce UX of users - -### Solution - -A circuit breaker acts as a proxy for operations that might fail. The proxy should monitor the number of recent failures that have occurred, and use this information to decide whether to allow the operation to proceed, or simply return an exception immediately. - -The proxy have 3 state: - -- Closed: The request from the application is routed to the operation. Count the number of recent failure, if the failure count more than threshold => Set state to Open. -- Open: The request/msg from the application fails immediately and an exception is returned. Start a timeout, when timer come, => set state to Half-Open -- Half-Open: A limited number of request sent to the service, if those are OK. Seem the service is OK => Set state to Close. - -### Considerations - -When using the circuit breaker, we might need to care about: - -- Exception Handling -- Logging -- Recoverability -- Testing Failed Operations -- Manual Override - -![[circuit-breaker-diagram.png]] - ---- - -#### Reference - -- https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker -- https://martinfowler.com/bliki/CircuitBreaker.html diff --git a/notes/Command Query Responsibility Segregation (CQRS).md b/notes/Command Query Responsibility Segregation (CQRS).md deleted file mode 100644 index cd0bc300a..000000000 --- a/notes/Command Query Responsibility Segregation (CQRS).md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - cqrs - - engineering - - pattern -redirect: - - /UAbbdA ---- - -### What - -- a pattern that separates read and update operations for a data store - -### Why - problem and context - -- Large, heavy DB load system. In big system, there is a huge number of db access (Read/Write). It could have bad effect to performance and data consistency. -- In traditional architectures, the same data model is used to query and update a database. That's simple and works well for basic CRUD operations. In more complex applications, however, this approach can become unwieldy. -- Read and Write data are often asymmetrical. - -### How - to solve the problem - -- CQRS separates reads and writes into different models, using commands to update data, and queries to read data. - - - Commands should be task-based, rather than data centric. ("Book hotel room", not "set ReservationStatus to Reserved"). - - Commands may be placed on a queue for asynchronous processing, rather than being processed synchronously. - - Queries never modify the database. A query returns a DTO that does not encapsulate any domain knowledge. - -- For greater isolation, you can physically separate the read data from the write data. In that case, the read database can use its own data schema that is optimized for queries. The read store can be a read-only replica of the write store, or the read and write stores can have a different structure altogether. - -### Benefit - -- Independent scaling. CQRS allows the read and write workloads to scale independently, and may result in fewer lock contentions. -- Optimized data schemas. The read side can use a schema that is optimized for queries, while the write side uses a schema that is optimized for updates. -- Security. It's easier to ensure that only the right domain entities are performing writes on the data. -- Separation of concerns. -- Simpler queries - -### Considerations - -- Complexity -- Messaging -- Eventual consistency: Read and Write DB need to be consistent - ---- - -#### Reference - -- https://docs.microsoft.com/en-us/azure/architecture/patterns/cqrs -- https://martinfowler.com/bliki/CQRS.html diff --git a/notes/Contract Context in Blockchain.md b/notes/Contract Context in Blockchain.md deleted file mode 100644 index 9160360f1..000000000 --- a/notes/Contract Context in Blockchain.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - blockchain -redirect: - - /ytcbpg ---- - -- When contract is deployed, it will always be immutable (cannot be edited). But in real life, there will always be problems to edit that contract (fix bugs, upgrade). -- Normally when you want to change, the flow will be: deploy new contract -> move the old contract state to the new contract -> beg the end-user to change to the new contract. -- There is another approach, proxy patterns, which is mainly about architecture contracts to achieve the goal of being able to update the contract after it has been deployed to the chain. -- This will basically have 1 contract as Proxy (this contract immutable) config dc 1 target contract (real contract handle logic). The end user will always connect to the proxy contract, the proxy contract will use a technique called `delegatecall` to fwd it to the target contract. -- When I need to change the contract code, I just need to deploy the new contract to the chain, configure the proxy contract call to the new contract. - ---- - -#### Reference - -- https://blog.openzeppelin.com/proxy-patterns/ diff --git a/notes/Convert subfolder into Git submodule.md b/notes/Convert subfolder into Git submodule.md deleted file mode 100644 index 59213d039..000000000 --- a/notes/Convert subfolder into Git submodule.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - git -redirect: - - /cJ6q3g ---- - -Clone new repository - -```shell -$ git clone --no-hardlinks original-repo copied-repo -``` - -Filter out the files you want to keep and remove the others - -```shell -$ cd copied-repo -$ git filter-branch --subdirectory-filter sub/module/path HEAD -- --all -$ git reset --hard -$ git gc --aggressive -$ git prune -$ git remote rm origin -``` - -Push the new repositories to the upstream server - -```shell -$ git remote add origin git@github.com:korya/submodule-repo.git -``` - -Add the new repository as submodules to the original repository - -```shell -$ cd original-repo -$ git rm -r sub/module/path -$ git commit -m "Removing the folders that are now repositories" -$ git submodule add git@github.com:korya/submodule-repo.git sub/module/path -$ git submodule init -$ git submodule update -$ git add .gitmodules sub/module/path -$ git commit -m "Added in submodules for removed folders" -``` - ---- - -#### Reference - -- https://gist.github.com/korya/9047870 diff --git a/notes/Data ERD.md b/notes/Data ERD.md deleted file mode 100644 index d4bd7beaf..000000000 --- a/notes/Data ERD.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - data - - diagram -redirect: - - /C8UYgg ---- - -### What - -ERD stands for Entity Relationship Diagram, a structural diagram that contains symbols nad connectors that visualizes 2 types of information - -- The major Entities -- The Relationship between those entities - -When we talk bout Entities, we are referring to the business objects within the system: people/role, tangile object (product). - -When we talk about Relationship, we are referring how those entities relate with each other. - -### Primary key & foreign key - -- Primary key: Use to uniquely define a record in database table -- Foreign key: a foreign key of a table is used aas a navigation to a primary key of another table - -### When - -ERDs are mostly developed during concept visualization, database design, database debugging, requirement information system. - -### How-to - -- Answer the purpose: What is this ERD for -- Clarify the model scope. This prevents the model from having too many redundant entities. Onyl select the entities that is critical and relevant to the business model. -- Define the models/ major entities -- Define the attribute of that entities (related information) -- Decide the relationship between those entities (which one might affect one another) -- Decide the kind of relationship - - 1:1 - - 1:many - - many:1 - - many:many - ---- - -#### Reference - -- [Explaning the ERD](https://www.visual-paradigm.com/guide/data-modeling/what-is-entity-relationship-diagram/#erd-data-models-conceptual) -- [Difference of the relationship](https://stackoverflow.com/questions/3113885/difference-between-one-to-many-many-to-one-and-many-to-many) -- [Example of ERD](https://www.guru99.com/er-diagram-tutorial-dbms.html) diff --git a/notes/Data Lifecycle.md b/notes/Data Lifecycle.md deleted file mode 100644 index 13a361220..000000000 --- a/notes/Data Lifecycle.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - data -redirect: - - /8J6nCg ---- - -#### Explaining - -Like human, data goes through a circle of life. from the derive until the collapse. - -- capturing: raw data get acquired from multiple sources -- maintaining: edit, format, label > turn raw/ disorganized data into consistent system -- using: extract the benefit from data. generate insight, visulaize results, explain decision. -- publishing: make data visible on different platforms, as a part of business // in-use -- archiving: storing for future use // historical data // reference -- deleting: as simple as it sounds. when data is no longer needed, it gets removed from the system. - -#### Why - -By knowing which stage the data is currently in, we get to decide what's best to act during the phase, how to protect and get the most out of it. - ---- - -#### Reference - -https://www.thinkautomation.com/eli5/the-data-lifecycle-explained/ diff --git a/notes/Database Sharding.md b/notes/Database Sharding.md deleted file mode 100644 index b8d989d23..000000000 --- a/notes/Database Sharding.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - database - - sharding - - engineering -redirect: - - /2RFXmg ---- - -### Why? - -Data-driven app need ability to scale dynamically to against significant growth in the future - -### What? - -- Database architecture -- Horizontal partitioning - separating one table’s rows into multiple different tables (partitions) - -### How? - -- Breaking up one’s data into 2 or more smaller chunks (logical shards) -- Chunks are distributed across separate DB nodes (physical shards) -- Exemplify a shared-nothing architecture - means that they don’t share any of the same data or computing resources -- A table can be replicate to all shared as a ref table -- Can be implement in both BE layer and DB layer (clickhouseDB) - -### Benefit - -- Facilitate horizontal scaling (scaling out) by adding more machines to an existing stack in other to spread out the load and allow for more traffic and faster processing <> scaling up - add more compute power by upgrade hardware -- Ignore limitation of non-distributed database in terms of storage and compute power. -- Increasing speed of query stmt (each shard query itself instead of all rows) -- Avoid the the impact of outages (an outage is likely to affect only a single shard) - -### Limitation - -- Complexity of shared DB architecture implementation -- Manage data across multiple shard locations instead of one point -- Shards eventually become unbalanced -- Difficult to return to its unsharded architecture -- Sharding isn’t natively supported by every database engine - -### Sharding architectures - -- Key Based Sharding: hashing a value from new data to determine which shard the data should go to -- Range Based Sharding: Using range of given value to determine target shard to store data -- Directory Based Sharding: Implement a lookup table for shard system. diff --git a/notes/Deno.md b/notes/Deno.md deleted file mode 100644 index f30cf3083..000000000 --- a/notes/Deno.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: null -description: null -date: null -author: Giang Ngoc Huy -tags: - - engineering - - javascript - - typescript -redirect: - - /yMLPGA ---- - -## What - -- A new JavaScript runtime, written in Rust (JsVM 💃🏻) -- Built to follow the standard Web API -- Take advantage of TypeScript, WASM and ESM module (and support them natively) -- "Node" written weirdly backwards - -## Why - -![](https://i.imgur.com/kFo8ECL.png) - -- Deno follows the standard Web API [and is leading the industry in implementing them](https://github.com/denoland/deno/pull/11941) - - This is with the aim to make JS/TS project ideally more universal -- secure by default - - requires `--allow-*` flags to enable specific features like read/write, network access, etc... - - This is so a random project/module can't read your file system unless you **explicitly** allow it to -- no need for `npm install`, pre-bundling (e.g: `tsc`/`webpack`/`package.json`) dances and rituals -- Perfect to deploy on modern platforms like Cloudflare Workers and [Deno deploy](https://deno.com/deploy/docs) - -## Why not? - -- Although native ESM and `script type="module"` is widely supported in modern browsers, some very new APIs like `URLPattern` is not completely supported everywhere -- dependencies on Deno are imported by URL, so it makes it somewhat messy, but [`import map`](https://deno.land/manual/linking_to_external_code/import_maps) can be used to improve readability - - `import lodash from "https://deno.land/x/lodash@4.17.19"` - - `deno` will download available imports online and cache locally before running - -## What about Node.js? - -- Node.js and its ecosystem has grown into a messy place -- History of inconsistent authoring/bundling [formats](https://dev.to/iggredible/what-the-heck-are-cjs-amd-umd-and-esm-ikm) (cjs, amd, umd) -- Node 16 adds ESM support, but before moving to ESM format, 2 things needed to happen: - - the library author has to [break backward compatibility](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#pure-esm-package) - - library users have to [migrate their project to an ESM format](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#pure-esm-package) - - just imagine the whole Node.js ecosystem going through this 💥 -- Confusion with JavaScript in the browser (e.g: `require` vs `import`) -- Non-standard & incompatible with modern JavaScript - -## Extras - -Deno also removes one most confusing part when developing JavaScript: build tools. - -Webpack/Babel and their bundling process is the most over-engineered (albeit necessary) part of JavaScript development. Just needing to know how bundlers work to effectively leverage them is a crucial part of writing & authoring JavaScript projects/libraries, and has been over the recent decade. - -Node.js developers often need to work with Webpack & Babel just to have the convenience to write their code in modern JavaScript (e.g: `import` vs `require`) or be able to work with TypeScript. - -Deno, on the other hand, strives to move along with the modern JavaScript standard by defaulting to ESM, with native support for all standard APIs and TypeScript/JSX, removing the need to rely on build tools. - -> "Now we can sit down and write actual JavaScript instead of spending half a day initialize new project with Webpack, Babel, JSX and another half to configure Jest to read our `.babelrc` config. _Such. fresh. air._" - -### Deno deploy - -Similar to Cloudflare Workers, Deno deploy is a serverless deployment platform using the Service Worker API to create deployments, currently in (free) public beta. - -What's really cool about Deno deploy is that it only requires a link to your entry file to deploy, and deploy time is super fast. - -![](https://i.imgur.com/GygtLWs.png) - -Interesting projects to follow: - -- https://github.com/exhibitionist-digital/ultra -- https://github.com/lucacasonato/fresh diff --git a/notes/Domain-Driven Design.md b/notes/Domain-Driven Design.md deleted file mode 100644 index 45e10b72a..000000000 --- a/notes/Domain-Driven Design.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - ddd - - design -redirect: - - /To0Zbw ---- - -Why Domain Driven Design: Using the same business term makes it easier for designers, business experts land developers to communicate and stay on the same page during design and development phase. - -DDD helps to remove roadblock between devenand designs, stay consistent in workflow. - ---- - -Source: https://builtin.com/software-engineering-perspectives/domain-driven-design diff --git a/notes/Email in dev's PoV.md b/notes/Email in dev's PoV.md deleted file mode 100644 index 5f05fb7af..000000000 --- a/notes/Email in dev's PoV.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - development - - email -redirect: - - /voI2Mg ---- - -2 types that draws the most focus - -- marketing: newsletter, product launch, feature updates, - -> promotional purpose - -> users won't know when they will receive -- transaction: direct controlled by dev - - generated and sent when a `transaction` happen // trigger by action - - form submission, purchase invoice, password reset - -> users want to receive, actively know when - -A-Z walkthrough for email development - ---- - -Source: https://hackernoon.com/the-a-z-of-email-development-and-sending-d9w33h4 diff --git a/notes/Garbage collector in Elixir.md b/notes/Garbage collector in Elixir.md deleted file mode 100644 index 176997d5e..000000000 --- a/notes/Garbage collector in Elixir.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir - - garbage -redirect: - - /IZ8cmQ ---- - -BEAM process has its own stack and heap for small data binaries(<64bytes) Larger binaries are stored in a shared memory space with a ref-counted pointer(called ProcBin) that lives in a Process's heap. There are many data stacks and heaps in our application, one per process. - -- Generational - minor garbage collection. This method reclaims **heap memory** that is no longer referenced by the process. Idea: most memory is only used for a short period of time. It is **highly performant**, but looks at **recently allocated** - -- Full-sweep - major garbage collection. This method looks at the entire heap of a process, and reclaims as **much memory as possible**. Idea: memory must be clean when the heap is close to being full. It is fast in practice, but it can be slow if executing on a process that is actively holding a lot of data - -### How to collect data manually - -1. From OTP 20 provide a function to reclaims the memory in a process: :erlang.garbage_collect/1. We can use :erlang.garbage_collect/0 to trigger garbage collection for the process that invoked the function -2. Passing the :hibernate_after option for GenServer.start/3 function. The data will be reclaims when the process is idled. Ex: Phoenix Channel use hibernate_after to enter hibernation 15 seconds after processing their last message. -3. Returning :hibernate at the end of the GenServer callbacks - -```elixir -def handle_call(:clear, _from, _state) do - {:reply, :ok, [], :hibernate} -end -``` diff --git a/notes/Graphql.md b/notes/Graphql.md deleted file mode 100644 index 9ea10f0e2..000000000 --- a/notes/Graphql.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - api - - graphql -redirect: - - /N2IcwQ ---- - -A query language to write schema on backend server. Unlike SQL (query language for database), GraphQL follows a stricter format and gets utilized on API, which, usually gets compared to RESTful API. - -For query language for API from backend side. - -#### Pros - -- Retrieve the exact data based on pre-set schema (while REST API retrieves a full result which might contains irrelevant/ redundant data) -- Single request for multiple resources (REST API requires separate API for each table) -- Notify update when data is changed - -#### Cons - -- Rare on adoption -- Hard to pick up -- Inappropriate for constantly fectch data system - ---- - -#### Reference - -https://exarhouleasjohn.medium.com/what-is-graphql-6dcc27a18914 diff --git a/notes/History of Regular Expression.md b/notes/History of Regular Expression.md deleted file mode 100644 index c29c15219..000000000 --- a/notes/History of Regular Expression.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - regex -redirect: - - /gpAS8Q ---- - -_Buzz Andersen (BA) is an interesting builder on the internet (and has been for some time, with stints at Apple, Square, Tumblr, and more) but has also emerged as an important and moral voice in technology’s evolution. He’s an incredible [twitter follow](https://mobile.twitter.com/buzz) and I always feel smarter for having interacted with him. - Colin (CJN)_ - -_**Buzz here.**_ On March 10, 2021, many Russian Internet users suddenly found they were having trouble accessing parts of the web. Kentik, a US-based network analysis company, [reported](https://twitter.com/DougMadory/status/1369648537634545673) that network traffic to Rostelecom, the Russian state telecom, was down as much as 24%, and cybersecurity Twitter was abuzz with speculation about the causes. - -Eventually, a Russian artist and Twitter user named Gregory Khodyrev [realized](https://twitter.com/GregoryKhodyrev/status/1369658173334884360) what was going on: someone at Russia’s state Internet censor, Roscomnadzor, had attempted to block the Internet domain “t.co” (used by Twitter’s URL shortener), but had instead managed to cut off access to _any_ domain containing the text pattern “t.co.” This meant that sites such as “microsoft.com,” “reddit.com,” and even Russia’s own state media outlet “rt.com” were rendered suddenly inaccessible. - -Readers with a modicum of technical knowledge may already have an inkling of what likely happened here: some hapless censor, attempting to [curb Twitter’s political influence](https://www.nytimes.com/2021/03/10/world/europe/russia-twitter.html), installed a URL pattern matching rule on Russia’s [national firewall](https://www.theguardian.com/technology/2019/apr/28/russia-great-firewall-sovereign-internet-bill-keeping-information-in-or-out) that turned out to have been just a tad overzealous. The rule in question was almost certainly expressed using a notoriously abstruse notation called a “regular expression.” - -There is an old programmer joke (generally [attributed](http://regex.info/blog/2006-09-15/247) to Jamie Zawinski) that goes: “Some people, when confronted with a problem, think ‘I know, I'll use regular expressions.’ Now they have two problems.” In accidentally cutting off access to a large part of the Internet for an entire country (and in the process drawing embarrassing attention to Russia’s censorship efforts), our anonymous bureaucrat seems to have proven that old chestnut quite handily. - -**Why is this interesting?** - -A regular expression, or “regex,” is a kind of algebraic shorthand for specifying text patterns. Here, for example, is a regular expression that will match will match the words “sink,” “sank,” or “sunk.” - -> _s[aiu]nk_ - -The brackets indicate that either “a,” “i,” or “u” are acceptable matches for the second character in the series. - -Now, to make things slightly more interesting, we can add an optional element: - -> _st?[aiu]nk_ - -The question mark makes “t” optional in the pattern, and means our expression can now match not only “sink,” “sank,” and “sunk,” but also “stink,” “stank,” and “stunk.” - -Finally, to dramatically expand the number of text “strings” our expression can match, we can add a period (meaning “any character”) followed by an asterisk (meaning “as many as possible”). - -> _s.\*nk_ - -Now our regex can match “stonk,” “slink,” or indeed any other word starting with “s” and ending with “nk.” - -Most regular expression implementations also support a feature called “anchors,” which specify that the match must appear at the start or end of a text string. For example: - -> _^t.co$_ - -The above expression matches the pattern “t.co”—but only if the “t” is the first character in the string and the “o” is the last. Without the “^” and “$,” by contrast, the expression would, as our anonymous Russian censor learned, match the pattern no matter where it appears in the string. - -The concept of a regular expression has a surprisingly interesting history that dates back to the optimistic, mid-20th Century heyday of artificial intelligence research. - -The term itself originated with mathematician [Stephen Kleene](https://en.wikipedia.org/wiki/Stephen_Cole_Kleene). In 1943, neuroscientist Warren McCulloch and logician Walter Pitts had just [described](https://web.csulb.edu/~cwallis/382/readings/482/mccolloch.logical.calculus.ideas.1943.pdf) the first mathematical model of an artificial neuron, and Kleene, who specialized in theories of computation, wanted to investigate what networks of these artificial neurons could, well, theoretically compute. - -In a 1951 [paper](https://www.rand.org/content/dam/rand/pubs/research_memoranda/2008/RM704.pdf) for the RAND Corporation, Kleene reasoned about the types of patterns neural networks were able to detect by applying them to very simple toy languages—so-called “regular languages.” For example: given a language whose “grammar” allows only the letters “A” and “B”, is there a neural network that can detect whether an arbitrary string of letters is valid within the “A/B” grammar or not? Kleene developed an algebraic notation for encapsulating these “regular grammars” (for example, a*b* in the case of our “A/B” language), and the regular expression was born. - -Kleene’s work was later expanded upon by such luminaries as linguist Noam Chomsky and AI researcher Marvin Minsky, who formally established the relationship between regular expressions, neural networks, and a class of theoretical computing abstraction called “finite state machines.” - -[![](https://cdn.substack.com/image/fetch/w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ffb365073-fe9b-4c1b-a935-5d62ca57bd61_551x720.png)](https://cdn.substack.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ffb365073-fe9b-4c1b-a935-5d62ca57bd61_551x720.png) - -These discoveries made early AI pioneers feel as if they were unlocking the fundamental mysteries of the mind, and for a while it seemed as if true machine intelligence might be just around the corner. Unfortunately, this heady early enthusiasm for this work was soon to hit a brick wall. In 1969, Minksy published his [controversial](https://www.jstor.org/stable/285702) book [Perceptrons](https://www.amazon.com/Perceptrons-MIT-Press-Introduction-Computational/dp/0262534770?crid=23RVCCWWS4T6Z&dchild=1&keywords=perceptrons&qid=1622757363&s=books&sprefix=perceptrons%2Cstripbooks%2C157&sr=1-1&linkCode=ll1&tag=noahbrierdotc-20&linkId=e3c3406b9d166fde261f9f0bc0356f00&language=en_US&ref_=as_li_ss_tl), which explored some important limitations of early neural networks—with devastating consequences for the emerging science and its pioneers. As Brian Christian writes in his book [The Alignment Problem](https://www.amazon.com/Alignment-Problem-Machine-Learning-Values/dp/0393868338?_encoding=UTF8&qid=1622757405&sr=1-1&linkCode=ll1&tag=noahbrierdotc-20&linkId=6d2115b8f431857031f2d7895f8d766f&language=en_US&ref_=as_li_ss_tl): - -> _It is as if a dark cloud has settled over the field, and everything falls apart: the research, the money, the people. Pitts, McCulloch, and Lettvin, who have all three moved to MIT, are sharply exiled after a misunderstanding with MIT’s Norbert Wiener, who had been like a second father figure to Pitts and now won’t speak to him. Pitts, alcoholic and depressed, throws all of his notes and papers into a fire, including an unpublished dissertation about three-dimensional neural networks that MIT tries desperately to salvage. Pitts dies from cirrhosis in May 1969, at the age of 46.”_ - -The study of neural networks soon settled into a [long fallow period](https://en.m.wikipedia.org/wiki/AI_winter) as many researchers shifted their efforts away from “connectionist” theories (in which intelligence emerges from massively parallel networks) and toward a more paradigmatically familiar “symbolist” approach (which sought to develop intelligence by loading machines up with databases of facts and trying to teach them the rules of logic). The repudiation of neural nets was so thorough that it seems unlikely regular expressions would be so widely known outside of esoteric computer science today if it wasn’t for the efforts of [Ken Thompson](https://en.wikipedia.org/wiki/Ken_Thompson), one of the creators of the UNIX operating system at Bell Labs. - -Thompson started [applying regexes](http://www.oilshell.org/archive/Thompson-1968.pdf) to more mundane tasks by integrating them into the advanced search feature of his QED text editor in 1968. From there, regular expressions became a mainstay of UNIX, used to provide “wildcard” matching pretty much anywhere in the operating system that text search is required. In the 1980s, [Larry Wall](https://en.wikipedia.org/wiki/Larry_Wall) gave regexes another boost by making them a core feature of his text-oriented programming language, [Perl](https://en.wikipedia.org/wiki/Perl), which went on to become the indispensable “duct tape” of 1990s web development. This secured regular expressions a permanent place in the workaday webmonkey toolbox—and a key role in the kind of back-office sysadmin work Russia’s firewall administrators are tasked with. - -As for McCulloch, Pitts, Kleene, et. al, they eventually got their vindication. The early 2010s saw a resurgence of interest in neural networks, which in turn led to the advent of “deep learning” and massive advances in problems like facial recognition, machine translation, and autonomous vehicle navigation. So next time you’re puzzling through the inscrutable syntax for advanced search in your text editor, Internet search engine, or nationwide firewall, spare a thought for the once discredited AI pioneers who ended up making everything from wildcard matching to self-driving cars a reality. (_BA_) - -**Comic of the Day:** - -After yesterday’s Fastly outage, [this XKCD seems appropriate](https://xkcd.com/2347/). (_NRB_) - -![](https://cdn.substack.com/image/fetch/w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F794f1269-8ef9-4ef0-b324-e5743212053b_770x978.png) - -**Quick Links:** - -- [What is the 'magic' brake setting cost Hamilton in Baku – and why do Mercedes struggle at low-grip tracks?](https://www.formula1.com/en/latest/article.tech-tuesday-what-is-the-magic-brake-setting-that-cost-hamilton-in-baku-and.79Mw4XSrxksy7jen3CE8FX.html) (_NRB_) - -- [The design history of Athens iconic apartments](https://www.bloomberg.com/news/features/2020-07-15/the-design-history-of-athens-iconic-apartments) (_NRB_) - -- [Sponsored Link] If you’re at a SaaS company, check out [Variance](https://www.variance.com/). It’s a tool to help grow customers (some people are calling it a [PLG CRM](https://www.variance.com/land/plg-crm)). If you have questions or want to try it, [get in touch](https://www.variance.com/contact). (_NRB_) - ---- - -#### Reference - -https://whyisthisinteresting.substack.com/p/the-regular-expression-edition diff --git a/notes/How CAPTCHA works and why it's getting harder.md b/notes/How CAPTCHA works and why it's getting harder.md deleted file mode 100644 index 4ab52be33..000000000 --- a/notes/How CAPTCHA works and why it's getting harder.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: null -description: null -date: null -tag: captcha -redirect: - - /VNkBJw ---- - -CAPTCHA stands for _Completely Automated Public Turing test to tell Computer and Humans Apart_. CAPTCHA works whenever the computer asks user to indentify letters. Later then, the invention of reCAPTCHA requires human to select graphic based on topics, or enter images of text that computers have trouble deciphering. - -This helps driven the security barrier and more privacy protection. - -By providing CAPTCHA the right answers, human somehow feeding data for the computer to train and evolve, which is the reason why CAPTCHA is getting more complex. - -Once user behavior seems to resemble a bot's behavior, a test may be triggered and automatically ratchering up the difficulty as a higher defense mechanism. - -**Follow-up FAQs** - -1. What happened if CAPTCHA is too strong and people find it hard to break -2. Some opinion state that using physical privacy key (like YubiKey) will help, but how, and will it work - ---- - -Source: -https://www.vox.com/22436832/captchas-getting-harder-ai-artificial-intelligence diff --git a/notes/Install Elixir & Phoenix.md b/notes/Install Elixir & Phoenix.md deleted file mode 100644 index a49da4910..000000000 --- a/notes/Install Elixir & Phoenix.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir -redirect: - - /LbA0RQ ---- - -```shell -$ mix local.hex -$ mix archive.install hex phx_new -$ mix help phx.new -$ mix phx.new pod --no-html --no-webpack -``` diff --git a/notes/Maslow's hierarchy of needs.md b/notes/Maslow's hierarchy of needs.md deleted file mode 100644 index 60529962d..000000000 --- a/notes/Maslow's hierarchy of needs.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - design - - maslow - - ux -redirect: - - /eeOaKA ---- - -![](https://www.simplypsychology.org/maslow-5.jpg) - -Maslow's hierarchy of needs is a motivational theory in psychology comprising a five-tier model of human needs, often depicted as hierarchical levels within a pyramid. From the bottom of the hierarchy upwards, the needs are: physiological (food and clothing), safety (job security), love and belonging needs (friendship), esteem, and self-actualization.Needs lower down in the hierarchy must be satisfied before individuals can attend to needs higher up. In general, humans are made of 3 main components: **physical**, **psychological** and **consciousness**, Maslow's hierarchy of needs is a ladder to bring certainty for these components. - -In business, after analyzing target customers, Maslow's hierarchy of needs is applied to define needs to make suitable products or strategies. - -In facts, people don't always desire certainty, since too much certainty leads to boredom. Thus, people also desire change, adventure and variety to feel alive. Variety and certainty are very much connected to each other. They sit on either side of a scale that needs to be in constant balance. When we have too much certainty we reach a level where we are running on automatic. We look for a change to break up the monotony and bring some interest. This is where innovation comes from. As people live a boring life, keep doing things in their daily routine, they tend to do a lot of things to bring change and interest into their lives, some of them are innovating and creating new things. - ---- - -#### Related - -- [[Human needs]] -- [[Seven deadly sins]] -- [[Skandhas - the five aggregates]] - -#### Reference - -- [Maslow's hierarchy of needs](https://www.simplypsychology.org/maslow.html) -- [Certainty & variety](https://www.ronitbaras.com/emotional-intelligence/personal-development/six-human-needs-certainty/) diff --git a/notes/Memory safety in Rust.md b/notes/Memory safety in Rust.md deleted file mode 100644 index 874b494ed..000000000 --- a/notes/Memory safety in Rust.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - rust -redirect: - - /BKacLQ ---- - -The garbage collector of Golang helps developers by automatically freeing the memory of their programs when it is not needed anymore. However, keeping track of the memory and cleaning it could impact the performances of our programs. - -Meanwhile, Rust's mechanism in memory safety acts different. Dividing in `stack` and `heap`. It predicts when will the data be out of used and automatically drop (remove) it, which reduces the cost of memory management during runtime. - -The obsolence of garbage collector then enhance and speed up performance, which make this language become one of the top choices for 2021. - ---- - -#### Reference - -- https://stackoverflow.blog/2020/01/20/what-is-rust-and-why-is-it-so-popular/ -- https://serokell.io/blog/rust-guide -- https://blog.rust-lang.org/2021/05/11/edition-2021.html -- https://msrc-blog.microsoft.com/2019/07/22/why-rust-for-safe-systems-programming/ diff --git a/notes/Micro Frontends.md b/notes/Micro Frontends.md deleted file mode 100644 index 67962c9f7..000000000 --- a/notes/Micro Frontends.md +++ /dev/null @@ -1,183 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - micro-frontend -redirect: - - /lNA1ug ---- - -### What is Micro-frontend? - -An architectural style where independently deliverable frontend applications are composed into a greater whole - -**Benefits** - -- Smaller, more cohesive and maintainable codebases -- More scalable organisations with decoupled, autonomous teams -- The ability to upgrade, update, or even rewrite parts of the frontend in a more incremental fashion than was previously possible -- Easier maintenance: Keeping frontend repositories small and specialized allows them to be more easily understood, and this simplifies long-term maintenance and testing. - -### Micro-Frontend in actions - -#### Build-time integration - -- Publish each micro frontend as a package and have the container application include them all as library dependencies. - -```js -{ - "name": "@shop/container", - "version": "1.0.0", - "description": "E-commercial website", - "dependencies": { - "@shop/products": "^1.0.0", - "@shop/order": "^1.0.0", - "@shop/user-profile": "^1.0.0" - } -} -``` - -Cons - -- Have to re-compile and release every single micro frontend in order to release a change to any individual part of the product. - -#### Run-time integration via iframes - -- Use iframe to connect other micro frontends. - -```html - - - Shop - - -

Welcome to Shop

- - - - - - -``` - -Cons - -- Difficult to make page responsive -- Difficult to integrate between different parts of application, make routing, history, and deep-linking more complicated - -#### Run-time integration via JavaScript - -- Each micro frontend is included onto the page using a ` - - - -
- - - - -``` - -#### Run-time integration via Web Components - -- Define each micro frontend as an HTML custom element for the container to instantiate. - -```html - - - Shop - - -

Welcome to Shop

- - - - - - - -
- - - - -``` - -Cons - -- Don't support server-side rendering -- No 100% browser support - -#### Cross-application communication - -1. [Custom event](https://developer.mozilla.org/en-US/docs/Web/Events/Creating_and_triggering_events) -2. Cross-application communication via routing - -#### Serverside rendering / universal rendering - -- Server-side rendering is always tricky problem. -- Use [Server Side Includes](https://en.wikipedia.org/wiki/Server_Side_Includes) to plug in page-specific content from fragment HTML files: - -#### Cons of micro-frontend - -- Duplication of dependencies => increase payload size -- App performance -- Learning curve -- Only suitable for medium, large projects - ---- - -#### Reference - -- https://micro-frontends.org/ -- https://martinfowler.com/articles/micro-frontends.html -- https://www.tomsoderlund.com/programming/micro-frontends-a-microservice-approach-to-front-end-web-development -- https://github.com/ChristianUlbrich/awesome-microfrontends diff --git a/notes/Move an existing phoenix app under an umbrella app.md b/notes/Move an existing phoenix app under an umbrella app.md deleted file mode 100644 index 993a5b8e5..000000000 --- a/notes/Move an existing phoenix app under an umbrella app.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir -redirect: - - /IoCZVg ---- - -chrismccord [10:14 PM] @alanpeabody yes, it's straightforward - -1. mix new my_umbrella --umbrella -2. cd my_umbrella/apps -3. mv ~/path/to/phoenix_app ./ -4. Update the paths in your phoenix project's mix.exs to match: - ```elixir - def project do - [app: :my_app, - version: "0.0.1", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.2", - elixirc_paths: elixirc_paths(Mix.env), - compilers: [:phoenix, :gettext] ++ Mix.compilers, - build_embedded: Mix.env == :prod, - start_permanent: Mix.env == :prod, - aliases: aliases, - deps: deps] - end - ``` -5. shipit -6. if using brunch/node, update package.json paths that reference phoenix - -david.antaramian [10:16 PM] He described his problem more in `general` which is that he wants to keep his existing git history. Though, that should really just involve inverting what you suggested - -[10:18] @david.antaramian @alanpeabody ah yes, you should be able to do it similarly but within the existing repo - ---- - -#### Reference - -- https://elixir-lang.slack.com/archives/phoenix/p1472921051000134 diff --git a/notes/Nextjs Rewrites.md b/notes/Nextjs Rewrites.md deleted file mode 100644 index d82a5e714..000000000 --- a/notes/Nextjs Rewrites.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - nextjs -redirect: - - /NO5cJg ---- - -After Googling, I've come up with a solution - -Nextjs has a feature called `Rewrites`. Rewrites allows us to map an income request for a different destination request path. We're going to Rewrite it as below: - -````async rewrites() { - return [ - { - has: [ - { - type: "host", - value: "(?.*)", - }, - ], - source: "/", - destination: "/hosts/:host", - } - ]; - },``` -After applying Rewrite, every time a user logs in `.example.com` => The request will be rewritten as `example.com/hosts/`, we can also use `getStaticProps` (SG mechanism) to get a `username` => Query user info => Render html. => Improve performance - -P/s: This post refers to Next.js users. So for non-Next.js, it might be hard to understand. I found this a very interested topic to note down. I figure it might make a good reference. - ---- - -#### Reference - -- https://nextjs.org/docs/basic-features/data-fetching -- https://nextjs.org/docs/api-reference/next.config.js/rewrites -```` diff --git a/notes/OAuth 2.md b/notes/OAuth 2.md deleted file mode 100644 index fc1571bc7..000000000 --- a/notes/OAuth 2.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - oauth -redirect: - - /4zNpmQ ---- - -Traditional client-server authentication model: resource owner provides credentials for client to access resource - -### Problems & limitations - -- Store credentials on the third-party to reuse => stolen credentials risk -- Password authentication => weak -- Resource owner does not have ability to restrict accessible resource -- Could not revoke access for specific party -- Intermediary hold user credentials => lost protected info - -OAuth2 was born to resolve above problem - -### OAuth2 - -OAuth 2.0 is the industry-standard protocol for authorization. OAuth 2.0 focuses on client developer simplicity while providing specific authorization flows for web applications, desktop applications, mobile phones, and living room devices. - -#### Roles - -- Resource owner: server, end user - an entity capable of granting access to a protected resource -- Resource server: the server hosting the protected resources -- Client: the application making protected resource requests on behalf of the resource owner and with its authorization -- Authorization server: the server issuing access tokens to the client after successful authentication of the resource owner and obtaining authorization - -Note: The authorization server may be the same server as the resource server or a separate entity. A single authorization server may issue access tokens accepted by multiple resource servers. - -#### Authorization grant - -Have 4 types - -- Authorization code - - - Using an authorization server as intermediary between client and resource owner - - Authorization code is a temporary code that the client will exchange for an access token - - Authorization server authenticates the resource owner and obtains authorization - - client call to autho server => redirect to resource owner and verified owner with autho server => send autho code to client => exchange code + client secret to autho server => access token - - Resource owner's credentials are never shared with the client. - - Server to server communication. - -- Implicit - - Simplified authorization code flow - - Instead of issuing the client an authorization code, the client is issued an access token directly -- Resource owner password credentials - - User resource owner username and password directly -- Client credentials - - Each client has it own identifier and password. - - The authorization server MUST support the HTTP Basic authentication scheme for authenticating clients that were issued a client password. - - Or including the client credentials in the request-body using the following parameters: client_id and client_secret (NOT RECOMMENDED and SHOULD be limited) - -#### Access token - -- Credentials to access protected resource -- Opaque to the client -- Tokens represent specific scopes and durations of access -- Abstraction layer => replacing different authorization constructs (e.g., username and password) => more restrictive than authorization code => resource server does not need to understand a wide range of authentication methods. -- Access tokens can have different formats, structures, and methods of utilization (e.g., cryptographic properties) -- ttl is usually short, take advantage of refresh token => decrease ability to take by bad guy - -#### Refresh token - -- Refresh tokens are credentials used to obtain access tokens when - - Access token becomes invalid or expires - - Obtain additional access tokens with identical or narrower scope -- Issued to the client by the authorization server -- Refresh tokens are intended for use only with authorization servers and are never sent to resource servers - -#### Refresh token can be stolen - -To resolve this problem we have "The Proof Key for Code Exchange (PKCE, pronounced pixie) extension describes a technique for public clients to mitigate the threat of having the authorization code intercepted. The technique involves the client first creating a secret, and then using that secret again when exchanging the authorization code for an access token. This way if the code is intercepted, it will not be useful since the token request relies on the initial secret" - rfc7636 - -In March 2019, the OAuth 2.0 Security Best Current Practice deprecated the Implicit flow in favor of the Authorization Code flow with PKCE (Proof Key for Code Exchange). - -#### Why are Refresh Tokens considered insecure for an SPA? - -The refresh tokens are not used in SPAs, because in order to use it - and to get a new access token from the /token, the SPA needs to have a client secret, which cannot be stored securely in a browser. But since the OAuth 2.0 for Native Apps RFC recommends not requiring a client secret for the /token endpoint (for public clients), the refresh tokens could be used even in SPAs. - -How to resolve or decrease this problem? - -- Refresh token Rotation + BFF -- Silent authentication - -**Protocol Flow:** - -``` - +--------+ +---------------+ - | |--(A)- Authorization Request ->| Resource | - | | | Owner | - | |<-(B)-- Authorization grant ---| | - | | +---------------+ - | | - | | +---------------+ - | |--(C)-- Authorization grant -->| Authorization | - | Client | | Server | - | |<-(D)----- Access token -------| | - | | +---------------+ - | | - | | +---------------+ - | |--(E)----- Access token ------>| Resource | - | | | Server | - | |<-(F)--- Protected Resource ---| | - +--------+ +———————--------+ -``` - -**Refresh token flow ** - -``` - +--------+ +---------+ - | |--(A)------- Authorization grant --------->| | - | | | | - | |<-(B)----------- Access token -------------| | - | | & Refresh token | | - | | | | - | | +----------+ | | - | |--(C)---- Access token ---->| | | | - | | | | | | - | |<-(D)- Protected Resource --| Resource | | Auth | - | Client | | Server | | Server | - | |--(E)---- Access token ---->| | | | - | | | | | | - | |<-(F)- Invalid Token Error -| | | | - | | +----------+ | | - | | | | - | |--(G)----------- Refresh token ----------->| | - | | | | - | |<-(H)----------- Access token -------------| | - +--------+ & Optional Refresh token +———————--+ - -``` - ---- - -#### Reference - -- https://datatracker.ietf.org/doc/html/rfc6749 -- https://gaynwinters.wordpress.com/tag/user-agent-based-application/ -- https://www.oauth.com/oauth2-servers/server-side-apps/authorization-code/ -- https://portswigger.net/web-security/oauth/grant-types -- https://www.pingidentity.com/en/company/blog/posts/2021/refresh-token-rotation-spa.html -- https://stackoverflow.com/questions/54199636/what-is-the-difference-between-using-refresh-token-and-silent-authentication-for -- https://datatracker.ietf.org/doc/html/rfc7636 diff --git a/notes/Programming fundamental.md b/notes/Programming fundamental.md deleted file mode 100644 index d9dd36b09..000000000 --- a/notes/Programming fundamental.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - fundamental - - programming -redirect: - - /WxFTbg ---- - -The importance of programming fundamental is somehow being underrated. - -Ever wonder why big tech giants recruit their employees based on fundamentals and focus more on programming theory? - -### Why - -Because it's the foundation of everything. Coding skills can be easy to found on bootcamps or onine course (with practical perspective), but that's just it. They assist developers on resolving a task (like StackOverflow), but in the long run they don't do any favor in helping developers understand the core of programming, i.e, you have no idea what to do once the picture escalated. - -Regular companies are evaluating their candidates based on practical aspects (how to solve a task), that's why more and more people buy into online courses and spend money on certificates. That's why a lot of skillfuled CV failed althought they've resolved a dozen of use cases. They assume the foundation should belong to `academic`, and developers doesn't need to know these things. - -The fundamental affects how you grow. Short courses land you gigs, but fundamentals offers contexts where you get to explain, assess, analyst, and optimize the resolution. - ---- - -#### Reference - -https://swiftrocks.com/how-necessary-are-the-programming-fundamentals.html diff --git a/notes/Publisher Subscriber pattern.md b/notes/Publisher Subscriber pattern.md deleted file mode 100644 index a73f838da..000000000 --- a/notes/Publisher Subscriber pattern.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - microservice -redirect: - - /HPViwA ---- - -### Context - -- In cloud-based and distributed applications, components of the system often need to provide information to other components as events happen. -- Asynchronous messaging is an effective way to decouple senders from consumers, and avoid blocking the sender to wait for a response. - -### Solution - -Introduce an asynchronous messaging subsystem that includes the following: - -- An input messaging channel used by the sender. The sender packages events into messages, using a known message format, and sends these messages via the input channel. -- One output messaging channel per consumer. The consumers are known as subscribers. -- A mechanism for copying each message from the input channel to the output channels for all subscribers interested in that message. This operation is typically handled by an intermediary such as a message broker or event bus. - -### Some framework, library - -- Redis, RabbitMQ, Kafka -- Cloud solution: Azure (Service Bus, Events Bus, Event Grid), Google Cloud (Pub/Sub), AWS (Pub/Sub Messaging) - -### When to use - -- An application needs to broadcast information to a significant number of consumers -- An application can send information to consumers without requiring real-time responses from the consumers. -- An application needs to communicate with one or more independently-developed applications or services, which may use different platforms, programming languages, and communication protocols - -### Considerations - -When using the pattern, we might need to care about: - -- Subscription handling: consumer could subscribe/unsubscribe as need -- Security: Only authenticated service could access -- Subsets of messages: consumer could select what to subscribe, via topic or content filtering -- Wildcard subscribers: consumer could use wildcard to subscribe/unsubscribe -- Bi-directional communication: This pattern is treated as unidirectional, no 2-way contact -- Message ordering: The order in which consumer instances receive messages isn't guaranteed, and doesn't necessarily reflect the order -- Message priority -- Poison Message: should prevent message that have invalid data, or could break the service -- Repeated messages: The messaging infrastructure should detect and remove duplicated message -- Message expiration -- Message scheduling - -![[pubsub.png]] - ---- - -#### Reference - -- https://docs.microsoft.com/en-us/azure/architecture/patterns/publisher-subscriber -- https://hackernoon.com/publish-subscribe-design-pattern-introduction-to-scalable-messaging-781k3tae diff --git a/notes/SQL practices.md b/notes/SQL practices.md deleted file mode 100644 index eaf0fadb2..000000000 --- a/notes/SQL practices.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - data - - sql - - practice -redirect: - - /HZyIIQ ---- - -### Orm vs plain SQL - -- Raw: make all query by self from scratch -- ORM: Object relational mapping: programing technique for converting data between incompatible type systems using OOP -- SQL Generator: utility sql tool for generate Entity model, query. - -Factors affecting the technique decision - -- Productivity: spend time for development and maintain. Work with many different DB system, can replace in future -- Security: SQL injection -- Performance - -**Practice**: - -- Use ORM for almost case, with DEEP understanding the library. Raw is supported in ORM -- Use Raw and SQL Generator for some special case - -### Indexing - -The best technique for improve performance in database is to use indexes well. A index is a data structure that the database uses to correlate value to the rows where these values occur in a given column. An index provides an easy way for the database to find values more quickly than the brute-force method of searching the whole table from top to bottom. Index can also help an Update or Delete statement by finding the rows quickly (primary key) - -Misunderstanding when use indexes: - -- Index Aren't Standard: ANSI SQL standard says nothing about indexes. -- Defining no indexes or not enough indexes -- Defining too many indexes or indexes that don't help -- Running queries that no index can help - -Using INSERT, UPDATE, DELETE, the database has to update the index data structures for that table to be consistent so that our subsequent searches use these indexes to find the right set of rows reliably. - -There's no benefit to creating indexes that we don't use: - -- index on primary key column -- index for a long string type -- not going to search for specific values(datetime) -- make compound indexes that are redundant or seldom used (can use join, search, sorting order criteria) - -**Practice** - -Indexing base on the requirement or what queries are important to optimize - -- Measure the application code to find out the bottle-neck. Don't make informed decisions without information. Tool: pgfouine -- Using Explain to make a report of query analysis - query execution plan(QEP). - -- Make Index with condition CREATE UNIQUE INDEX line_items_prod_var_null_idx ON line_items (product_id) WHERE variant_id IS NULL - -- Rebuild the index data: Over time, as we update and delete rows, the indexes may become fragmented overtime - SQL system tool - -### Rounding errors - -- Fractional Number(float, decimal) is common type in db: money, measurement: length, weight, capacity, temperature, time. -- infinite precision VS finite precision: 1/3 vs 0.33 -- IEEE 754 represents floating-point numbers in a base-2 format. The Float data can't represent exactly in binary. The reason is we try to convert base-10 to base-2 https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html - -Practice: - -- Use Numeric or Decimal in SQL for fixed-precision fractional numbers - -### Query practice - -- Use alias make the more readable query -- Prefer = than like than like 'prefix%' than like '%content%' -- Using WITH for complex query -- Avoid DISTINCT, should verify the reason of duplicated record -- Avoid Select \*, spell out all columns you need(Select, Insert) - -Get row with Greatest value per group. Follow the Single-Value rule to build the query. The rows in each group are those rows with the same value in the column or columns you name after GROUP BY. Every column in the select-list of a query must have a single value row per row group. - -### Search a simple text - -- Optimize search text in db -- Poor performance solution: wildcard (%) that matches zero or more characters. - -**Practice:** - -- full-text search: support from db -- third-party search engines - right tool for the right job -- implement from scratch: XXX, Keywords, XXXKeywords. Create a procedure for searching. Make a trigger for update Keywords and XXXKeywords when Update or Insert data to XXX table - ---- - -#### Reference - -- https://pragprog.com/titles/bksqla/sql-antipatterns/ -- https://www.enterprisedb.com/blog/postgresql-query-optimization-performance-tuning-with-explain-analyze -- https://www.technical-recipes.com/2011/ieee-754-floating-point-to-binary-conversion/ diff --git a/notes/Saga Pattern.md b/notes/Saga Pattern.md deleted file mode 100644 index 45c785a0c..000000000 --- a/notes/Saga Pattern.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - microservice - - pattern -redirect: - - /XW4BqA ---- - -The saga design pattern is a way to manage data consistency across microservices in distributed transaction scenarios. - -### Context - -- You have applied the Database per Service pattern. Each service has its own database. Some business transactions, however, span multiple service so you need a mechanism to implement transactions that span services. -- For example, let’s imagine that you are building an e-commerce store where customers have a credit limit. The application must ensure that a new order will not exceed the customer’s credit limit. Since Orders and Customers are in different databases owned by different services the application cannot simply use a local ACID transaction. - -### Solution - -- Implement each business transaction that spans multiple services is a saga. A saga is a sequence of local transactions. -- Each local transaction updates the database and publishes a message or event to trigger the next local transaction in the saga. If a local transaction fails because it violates a business rule then the saga executes a series of compensating transactions that undo the changes that were made by the preceding local transactions. - -- There are two ways of coordination sagas: - - Choreography - each local transaction publishes domain events that trigger local transactions in other services - - Orchestration - an orchestrator (object) tells the participants what local transactions to execute - -### Resulting context - -- Benefit: It enables an application to maintain data consistency across multiple services without using distributed transactions -- Drawback: The programming model is more complex - -### When to used - -- Ensure data consistency in a distributed system without tight coupling. -- Roll back or compensate if one of the operations in the sequence fails. - -![[From_2PC_To_Saga.png]] - ---- - -#### Reference - -- https://microservices.io/patterns/data/saga.html -- https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/saga/saga -- https://blog.couchbase.com/saga-pattern-implement-business-transactions-using-microservices-part/ diff --git a/notes/Scenarios using concurrency in Elixir.md b/notes/Scenarios using concurrency in Elixir.md deleted file mode 100644 index f1d5c249e..000000000 --- a/notes/Scenarios using concurrency in Elixir.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - concurrency - - elixir -redirect: - - /PbhRfw ---- - -### Fire and forget - -Tell the system to execute some code asynchronously and not care about when it finishes, nor the result. - -Solution: `Task` abstraction in elixir. Tasks are processes meant to run a single action within their life-cycle - -Practice: - -Run a one-off action in Elixir to spawn the tasks under a supervision tree. We can easier to manage the processes(clean) - -```elixir -Task.Supervisor.start_child/2 -``` - -The default strategy for the Task.Supervisor is `:temporary` - never restarted. - -Implementation: - -- In the `Application` we add a `Task.Supervisor`, is named such as: AwesomeTaskSupervisor -- When we spawn a process, we start a child task with AwesomeTaskSupervisor name. - -```elixir -Task.Supervisor.start_child(AwesomeTaskSupervisor, process_func) -``` - -### Fan-in/fan-out - -Tell the system to execute consists of multiple operation. We run asynchronously, but care about the results. Example: uploading a bunch of docs to S3, sending a batch of emails to people. - -Solution: Task without using a supervisor - -Implementation: - -- Using Enum.map/2 and Task.async/1 to generate the processes. -- Using Enum.map/2 and Task.await/1 to wait all of processes are done - -### Scheduling work - -Tell the system run some work periodically every N minutes/hours/etc. - -Solution: GenServer abstraction. We save the period rule in the genserver. Using `Process.send_after/3` to trigger the `handle_info/2` processing function. - -Library: [Quantum](https://github.com/quantum-elixir/quantum-core) - -### Caching data - -Tell the system cache the frequently data. Such as: caching the frequently response, temperature data. - -Solution: - -- Agents abstraction: a single process, therefore, we got bottleneck -- ETS: faster, support concurrency reads and writes -- GenServers - -Library: https://github.com/sasa1977/con_cache - ---- - -#### Reference - -- https://gist.github.com/hieuphq/f949e2a7074b0f23edb5f7dee2ad2533 -- https://functional.works-hub.com/learn/elixir-practical-concurrency-3794f diff --git a/notes/Semantic versioning.md b/notes/Semantic versioning.md deleted file mode 100644 index a6eb63483..000000000 --- a/notes/Semantic versioning.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - practice - - versioning -redirect: - - /zjBkqw ---- - -Versioning system that represents the product version, using a formal convention based on 3 parts in alpha numerics. - -#### Format - -A 3-component number in the format of "X.Y.Z", in which - -- X: Major version -- Y: Minor version -- Z: Patch version - -#### The purpose of SemVer - -Semantic Versioning represents changes and bug fixes. It shows how software moves closer to release target through different development phases. It tracks what's going on with the software as new plugins, add-ons, libraries and extensions are being built almost everyday. - -#### In software dependency context - -We have a definition of dependency software, means a software depends on another software. Using SemVer as rule of version release will better help manage dependency. - -For example A website depends on Hugo platform. In the event that these twos both follow Semver, when Hugo updates from version 4.2.0 to 4.2.1, you'll know for sure it's not a breaking change, you'll be able to update the website dependency without thinking twice. - -#### How SemVer works - -- Major version number: For incompatible API changes. -- Minor version number: Roll-out new backwards-compatible features. -- Patch version number: Introduce backwards-compatible bug fixes. - -#### Note - -- The first version starts at 0.1.0 and not at 0.0.1, since no bug fixes there. We start off with a set of features as the first draft. -- What's done before 1.0.0 is only the development phase. That's where you focus on making the product usable. - ---- - -#### Reference - -- https://www.researchgate.net/publication/333333382_What_Do_Package_Dependencies_Tell_Us_About_Semantic_Versioning -- https://docs.npmjs.com/about-semantic-versioning diff --git a/notes/Seven deadly sins.md b/notes/Seven deadly sins.md deleted file mode 100644 index 2cbb314fa..000000000 --- a/notes/Seven deadly sins.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - design - - ux -redirect: - - /8AavRQ ---- - -The seven deadly sins, also known as the capital vices, or cardinal sins, is a grouping and classification of vices within Christian teachings. Behaviors or habits are classified under this category if they directly give rise to other immoralities. According to the standard list, they are pride, greed, wrath, envy, lust, gluttony, and sloth, which are contrary to the seven heavenly virtues. These sins are often thought to be abuses or excesses of natural faculties or passions (for example, gluttony abuses the natural hunger for nourishment. - -- **Lust**: It is usually thought of as intense or unbridled sexual desire, lust could also mean other forms of unbridled desire, such as for money or power. -- **Glutton**: is the excess and overconsumption of anything to the point of waste. -- **Greed**: also known as avarice, cupidity, or covetousness. Greed is applied to an artificial, rapacious desire and pursuit of material possessions. -- **Sloth**: refers to a peculiar jumble of notions, dating from antiquity and including mental, spiritual, pathological, and physical states. It may be defined as absence of interest or habitual disinclination to exertion. -- **Wrath**: can be defined as uncontrolled feelings of anger, rage, and even hatred. Wrath often reveals itself in the wish to seek vengeance. -- **Envy**: can be described as a sad or resentful covetousness towards the traits or possessions of someone else. -- **Pride**: it is identified as dangerously corrupt selfishness, the putting of one's own desires, urges, wants, and whims before the welfare of other people. - -We often hear CEOs share that their products provide some values for users such as, convenience, optimization, sharing, connectivity ... In essence, what they do is to fulfill one or some of 7 needs above. For example, Instagram is not only a social media based on photo sharing, where users share their photos and show off their lifestyles; it also satisfies the needs of envy and pride of humans. - ---- - -#### Related - -- [[Human needs]] -- [[Maslow's hierarchy of needs]] -- [[Skandhas - the five aggregates]] - -#### Reference - -- [Seven deadly sins](https://en.wikipedia.org/wiki/Seven_deadly_sins) diff --git a/notes/Sharding pattern.md b/notes/Sharding pattern.md deleted file mode 100644 index b75bf4ef0..000000000 --- a/notes/Sharding pattern.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - pattern -redirect: - - /iF0lVg ---- - -### What - -This advanced pattern focuses on horizontally scaling data through sharding. - -### Context - -A centralized database could have problem: - -- Storage space: huge volume of data that could increase significantly over time. -- Computing resources: Number of concurrent users could be overwhelming. A single db could reach the limit. -- Network Bandwidth -- Geography: Sometime, due to laws, we have to put data in the country. E.g: Vietnam, Chinese - -Vertically scaling can only the temporary solution for big system. - -### Solution - -Divide the data store into horizontal partitions or shards. Each shard has the same schema, but holds its own distinct subset of the data. A shard is a data store in its own right (it can contain the data for many entities of different types), running on a server acting as a storage node. - -Separating one table’s rows into multiple different tables, known as partitions. Each partition has the same schema and columns, but also entirely different rows. - -Benefit of Sharding pattern: - -- Horizontal scaling, scaling is adding more shards, not required super strong hardware. -- Reduce contention and improve performance by load balancing between shards. -- In the cloud, shards can be located physically close to the users - -Each shard should have the same data-dividing algorithm. Where the data is stored should base on one or more attributes of data, should be static and not depend on data that might change. - -When application access the data, the sharding logic should direct to the necessary shards. Oftentimes, sharding is implemented at the application level, meaning that the application includes code that defines which shard to transmit reads and writes to. However, some database management systems have sharding capabilities built in, allowing you to implement sharding directly at the database level. - -### Sharding strategies - -- The Lookup strategy: must create and maintain a lookup table that uses a shard key to keep track of which shard holds which data. In a nutshell, a lookup table is a table that holds a static set of information about where specific data can be found. E.g: Data with region code is 1 (Asia) data should be store in Asia's shard. -- The Range strategy involves sharding data based on ranges of a given value. To illustrate, let’s say you have a database that stores information about all the products within a retailer’s catalog. You could create a few different shards and divvy up each products’ information based on which price range they fall into -- The Hash strategy involves using a value taken from newly written data — such as a customer’s ID number, a client application’s IP address, a ZIP code, etc. — and plugging it into a hash function to determine which shard the data should go to - -Each strategy should suite with specific requirements, we should carefully choose based on business/system logic. - -### When should you shard - -- The amount of data would exceed the storage capacity of single database node. E.g: Ethereum -- The volume of writes or reads to the database surpasses what a single node or its read replicas can handle, resulting in slowed response times or timeouts -- Huge bandwidth that would outpaces the bandwidth available for single node - -=> In very very big system should use shard - -### Considerations - -- Huge complexity -- Unbalance sharding -- No way back: After you sharding, really difficult to comeback -- Isn’t natively supported by every database engine - -![[DB_image_1_cropped.png]] - ---- - -#### Reference - -- https://www.digitalocean.com/community/tutorials/understanding-database-sharding -- https://docs.microsoft.com/en-us/azure/architecture/patterns/sharding diff --git a/notes/Shell Script.md b/notes/Shell Script.md deleted file mode 100644 index 19b2b542f..000000000 --- a/notes/Shell Script.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - development - - practice -redirect: - - /muW9bA ---- - -### Shell - -A program, takes command from the keyboard and give it to the OS -> execute task/ function. - -**Main function**: Interact with the operating system (OS) - -#### How human use shell program - -- Users: GUI (_graphical user interfaces_): button, icon, pointer, menu, scroll bar -> Anything that helps users to interact with the computer. -- Developers: CLI (_command line interfaces_): Interact with the computer through _terminal_: cd, ls, rm,... -> Text-based interfaces. - -_bash_ is an enhanced version of Shell program, the same with _ksh_, _tcsh_, _zsh_ - -### Shell script - -A list of commands that describes the steps that need to be executed. Those steps can be program execution, file manipulation or text printing. - -**To create Shell script** - -- open editor (vim/ vs code). name script file using .sh -- start the script with #!/bin/sh or #! /bin/bash -- write code - - my first script - - echo "Hello World, I'm practicing shell scripting" -- save as filename.sh -- executing: open terminal. type bash filename.sh - ---- - -#### Reference - -- -- -- diff --git a/notes/Skandhas - the five aggregates.md b/notes/Skandhas - the five aggregates.md deleted file mode 100644 index 931f281a4..000000000 --- a/notes/Skandhas - the five aggregates.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - process - - skandhas - - ux -redirect: - - /QwUtbw ---- - -![](http://payattention.ca/wp-content/uploads/2020/09/5aggregates-1080x675.jpg) - -**Skandhas** means "heaps, aggregates, collections, groupings". In Buddhism, it refers to the **five aggregates of clinging**, the five material and mental factors that take part in the rise of craving and clinging. They are also explained as the five factors that constitute and explain a sentient being’s person and personality. - -The five aggregates or heaps are: **form** (or material image, impression, **sensations** (or feelings, received from form), **perceptions**, **mental activity** or formations, and **consciousness**. - -- Form: matter, body or "material form" of a being or any existence. -- Sensation: sensory experience of an object. It is either pleasant, unpleasant or neutral. -- Perception: sensory and mental process that registers, recognizes and labels (for instance, the shape of a tree, color green, emotion of fear). -- Mental activity: all types of mental imprints and conditioning triggered by an object. Includes any process that makes a person initiate action or act. -- Consciousness: Awareness of an object and discrimination of its components and aspects. - -The five aggregates are often interpreted in the later tradition as an explanation of the constituents of person and personality. According to this interpretation, in each skandha – body, sensations, perceptions, mental formations and consciousness – there is emptiness and no substance. - ---- - -#### Related - -- [[Human needs]] -- [[Maslow's hierarchy of needs]] -- [[Seven deadly sins]] - -#### Reference - -- [What are the five skandhas] (https://www.lionsroar.com/what-are-the-five-skandhas/) diff --git a/notes/Software Architect and Software Design.md b/notes/Software Architect and Software Design.md deleted file mode 100644 index 4a5563f79..000000000 --- a/notes/Software Architect and Software Design.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - architecture -redirect: - - /QVRdGw ---- - -### Software Architecture (SA) - -**Sum up** - -- structure the software -- Answer for: what we are building - -**What** Software Architecture (SA) create a complete system architecture. - -The type of architecture will decide the constrains of software design. - -A software must have the charateristics (based on biz requirement). SA helps connects these characteristics into structural solution. - -SA prevents the accumulation of technical debts. - -**Example** - -- Microservices -- Monolith -- Event-Driven -- Serverless - -**How** The architecture descision must based on business requirement (scaling/ traffice volume, ...etc) Helps achieve biz goal and tech strategy - -### Software Design (SD) - -**Sum up** - -- implement the software -- answer for: how we are building - -**What** Software Design (SD) arranging the order of component is called software design. - -SD responsible for code-level design. - -- what each module is doing -- what problem does each module esolve - -**Example** - -- Factory pattern -- Adapter pattern - -**How** - -- Decide the function of each component and how they interact with others. -- Create the specifications of software artifacts. How individual modules/ components are amde and interact with each other -- Follows the SOLID principle - - each class have to serve one purpose - - open for extension. add more function without breaking the existing codes - - use inheritance and dont break the app's logic - - catergorize the interfaces <> structure the code so that a class won't be forced to implement a redundant function - - decoupling the code - ---- - -#### Reference - -- https://codeburst.io/software-architecture-the-difference-between-architecture-and-design-7936abdd5830 diff --git a/notes/Stale while revalidation.md b/notes/Stale while revalidation.md deleted file mode 100644 index dddf7b7e2..000000000 --- a/notes/Stale while revalidation.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - cache - - http - - practice -redirect: - - /njm1vg ---- - -### Concept - -- The **`Cache-Control`** HTTP header holds _directives_ (instructions) for caching (https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching) in both requests and responses. -- **`stale-while-revalidation`** is an extension **`Cache-Control`** directives which are not part of the core HTTP caching standards document. -- **`stale-while-revalidation`** indicates the client will accept a stale response, while asynchronously checking in the background for a fresh one. The _seconds_ value indicates how long the client will accept a stale response. Note that the time does not start at the time of the request itself, but, for example, after `max-age` has elapsed. -- Support for setting `stale-while-revalidate` alongside `max-age` in your `Cache-Control` response header is available in Chrome 75 (https://chromestatus.com/feature/5050913014153216) and Firefox 68(https://bugzilla.mozilla.org/show_bug.cgi?id=1536511). - -``` -Cache-Control: stale-while-revalidate= -``` - -### Example - -When you set the HTTP response header like below, this setting means - -```text -Cache-Control: max-age=1, stale-while-revalidate=59 -``` - -- If a request for the time is repeated within the next 1 second, the previously cached value will still be fresh, and used as-is, without any revalidation. -- If a request is repeated between 1 and 60 seconds later, then the cached value will be stale, but will be used to fulfill the API request. At the same time, "in the background," a revalidation request will be made to populate the cache with a fresh value for future use. -- If a request is repeated after more than 60 seconds, then the stale response isn't used at all, and both fulfilling the browser's request and the cache revalidation will depend on getting a response back from the network. - ---- - -#### Reference - -- https://web.dev/stale-while-revalidate -- https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control -- https://nextjs.org/blog/next-9-5 diff --git a/notes/Strangler Pattern.md b/notes/Strangler Pattern.md deleted file mode 100644 index 82f95a0d3..000000000 --- a/notes/Strangler Pattern.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - architecture - - strangler -redirect: - - /flYuWQ ---- - -### Context - -- We have a big old, obsolete running system, we want to replace it with new architecture, new tech stack. E.g: monolith => Micro service -- We don't have enough effort to replacing a whole new system at once, and that's also risky. We need a gradually migration. - -### Solution - -- Incrementally replace specific pieces of functionality with new applications and services -- Create a façade that intercepts requests going to the backend legacy system, then routes requests to either to the legacy application or the new services. Existing features can be migrated to the new system gradually - -### Consideration - -- At some point, when the migration is complete, the strangler fig façade will either go away or evolve into an adaptor for legacy clients. -- Make sure the façade doesn't become a single point of failure or a performance bottleneck - -### When not to use - -- When requests to the back-end system cannot be intercepted. -- Smaller systems where the complexity of wholesale replacement is low. - -![[strangler.png]] - ---- - -#### Reference - -- https://docs.microsoft.com/en-us/azure/architecture/patterns/strangler-fig -- https://dzone.com/articles/monolith-to-microservices-using-the-strangler-patt -- https://martinfowler.com/bliki/StranglerFigApplication.html diff --git a/notes/Technology Trend/How to follow the trend.md b/notes/Technology Trend/How to follow the trend.md deleted file mode 100644 index 23586cba8..000000000 --- a/notes/Technology Trend/How to follow the trend.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - technology -redirect: - - /FqjKpA ---- - -### Technology trend awareness - -Technology trend awareness as a skill refers to being mindful of the technology that is recently becoming popular and is readily accepted in the market or industry. It also encompasses one’s ability to recognize and understand the usefulness of any such technology for the success of his business. - -The industry-wide acceptance of a particular technology can be seen either as an opportunity or as a threat to an individual firm. Either way, the technology trend awareness skills serve as the most important asset. Ideal technology trend awareness skills would mean a person’s ability to either correspond to the industry-wide technology adoption or surpass it. - -- Increasing opportunities with new ideas and new ventures - ---- - -#### Reference - -- https://www.newgenapps.com/blog/how-to-stay-on-top-of-the-latest-business-technology-innovations/ -- https://www.cleverism.com/skills-and-tools/technology-trend-awareness/#:~:text=Technology diff --git a/notes/Technology Trend/Technology trend.md b/notes/Technology Trend/Technology trend.md deleted file mode 100644 index ebd65947d..000000000 --- a/notes/Technology Trend/Technology trend.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - technology -redirect: - - /jQOINg ---- - -Technology is evolving day by day and has a huge impact on both our individual and social lives. It plays a tool to solve our problems, helps improve the quality of lives and makes it much easier to live. As technology thrives, businesses must continually embrace technology trends to seize new opportunities and not fall behind. This study will learn what the technology trend is, how to evaluate new technology and how to follow the trend. - -### What is technology trend? - -Technology trends are new technologies that have the ability to **respond to high market demand**, and are researched by **many enterprises** for their **potential benefits,** with **a large room for growth.**. Thus, new technology will be considered trending when it satisfies all of the following 3 things: - -- High market demands. -- Great potential benefits. -- Opportunity for growth - -Market demands are changing day by day, based on the development of technology and the quality of life, subjective and objective events, and economic status. For example, during the Covid-19 pandemic, as residents were recommended to stay at home and limit interaction with each other, we have witnessed the rapid development of the delivery and shipping, collaboration platforms and videoconferencing for remote working, tele-health and telemedicine,... - -Needs always exist, and there are technological solutions to solve those needs. However, the size of the market is the key to determining the potential of the technology. For markets capable of bringing great benefits, both material and non-material, businesses and organizations will invest in the development of that technology. Therefore, always grasp technology trends that are not only important to the operation of a business, but also important to the individual working in the technology field. - -### Why do we need to follow the technology trend? - -Technology serves human needs, constantly evolving to meet new changing needs. Now, people have access to and become acquainted with technology quickly. In response to that speed, we need to constantly follow the advancement of technology, so as not to become out-of-date and, as a result left behind, then face extinction. We can look at the banking industry for an example. To adapt to the development of technology and human habits in using digital products, there is a race of digitalization in the banking industry. The traditional face-to-face interaction and physical card are now replaced by digital account, banking app, withdrawal by fingerprint, and live customer support via a kiosk. In order to survive in this market, all banks have invested a large amount of capital in following the technology trend to meet the needs of customers, which are convenience and safety. As a result, banks that continue to operate traditionally will fall behind, business deteriorates, and eventually disappears from the market. - -Besides staying relevant, competitive and retaining the current market, keeping up with technology trends creates more new opportunities. New technology will often lead to the creation of a new market. The sooner we catch up with the technology, the higher the market we occupy, the higher revenue. Individually, keeping up with the technology trends helps us increase self-confidence and credibility when consulting for clients. - -### How to evaluate a new technology - -Technology has a significant influence on life and is evolving, more and more new technologies are born day by day. However, not every technology is taken into a trend and is worth following. To know which trend we need to follow, we evaluate it through the following criteria: - -**Market size.** The market is made up of the needs of the customer (the user). To assess the size of the market, we need to review the urgency of that demand, the number or percentage of people with the same needs, and the market's ability to respond to that technology. - -**Development costs and the ability to profit from new technology.** Cost and profit are always the top issues that businesses are concerned about. Therefore, to consider whether to pursue it or not, we need to consider research, development, operational marketing costs and other related costs; as well as the affordability of the market for that product. - -**High applicability and development ability** Some technologies born to solve a certain problem but have no other applicability have gradually become obsolete and easily replaced. When human needs change rapidly according to the development of technology, the development ability of that technology must also be corresponding to be able to survive. A viable technology will expand the market and be more profitable. - -### How to follow the technology trends - -According to Cleverism, technology trend awareness as a skill refers to being mindful of the recent becoming popular technology and is readily accepted in the market or industry. It also encompasses one's ability to recognize and understand the usefulness of any such technology for the success of his business. To improve the technology trend awareness skill, there are some ways as below: - -- **Follow KOL on Social Media.** Twitter, Facebook, LinkedIn, even Reddit and Quora, or podcasts are great platforms to connect with experts worldwide. Follow and connect with them on social media platforms is the easiest way to know what's happening in this industry. Social media also lets us see what's trending at the exact moment in geographic areas. -- **Newsletter.** Warm-up a typical workday with a cup of coffee and a newsletter before deep burry in tasks is also a way to improve skill. Find some technology-focused websites and subscribe to their newsletters, the aggregated news will help you reduce the time searching information and identify trends quickly. Practicing reading habits every day allows us to recognize the technology movement. -- **Seminar and Community.** Reading is not the only way to gain knowledge, as there are some sources for which we are uncertain about the accuracy. So, after reading, try to discuss with people with the same interests and levels; or attend seminars, listen to experts directly, join in the discussion to express opinions and explore more. -- **Keep up with Competitors.** Learning from competitors is a great way to ensure we stay updated. Pursuing new technology is a race to grab the market pie, so learning and observing our competitors help us reposition, track and accelerate when necessary to avoid getting left behind. -- **Be open to change.** It is the most important thing when it comes to learning new technology. The change could be challenging and might be failed, but be flexible and open to changes, learn from the failure will keep us stay on the top of technology innovation. -- **Expand the area of focus.** It's not about the technology, it's an industry. Besides pursuing new technology, we need to learn more about the industry and its economic value. From the above evaluation criteria, we can make a judgment on that trend, avoiding the overwhelm of technology news. - -Finally, to broaden your understanding and embrace technology trends, stay open-minded. When we learn about new technologies to apply, we tend to be selective about the relevant and necessary information. For researching technology to become "knowledgeable", get all the information in an objective and broader way. diff --git a/notes/Technology Trend/Upcoming Trend.md b/notes/Technology Trend/Upcoming Trend.md deleted file mode 100644 index 8eff8ffd6..000000000 --- a/notes/Technology Trend/Upcoming Trend.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - technology -redirect: - - /mQiPC1M ---- - -## Artificial intelligence (AI) and machine learning - -Industry: ecommerce, finance, service, security, healthcare, supplychain, data-oriented industry - -[Ascent](https://www.ascent.io) - -- Builds, automates and manages compliance programs for financial firms, powered by cloud technology and artificial intelligence to keep programs powerful and secure over time. - -[Riskifited](https://www.riskified.com) - -- An AI powered platform that allows e-commerce sites to better identify legitimate shoppers and reduces the friction in the purchasing process to create a better customer experience. - -[Cylance](http://cylance.com/us/) - -- Cylance is a cyber security company that offers products built on a foundation of artificial intelligence, machine learning and algorithmic science, providing a data-driven approach to identifying and eliminating incoming threats. - -[Osaro](https://osaro.com/product) - -- Creates artificial intelligence technology that allows companies with industrial production lines to access high-performance robotic devices. The company’s software includes integrated perception and control capabilities that seamlessly accomplish daily tasks within industrial e-commerce and fulfillment facilities, and the company is working on new ways to grow its software to allow robots to grip and move irregularly shaped objects and make decisions. - -[Moveworks](https://www.moveworks.com/) - -- Offers a customer service interface that utilizes artificial intelligence and deep learning systems to help resolve customer IT support issues. Built on a framework of NLU and collective learning capabilities, Moveworks’ cloud-based platform allows organizations from startups to Fortune 500 companies to better serve employee needs in real time and reduce the strain of modern IT professionals. - -[Grammarly](https://www.builtinsf.com/company/grammarly) - -- An AI-enabled writing assistant that helps writers and communicators all over the world with spelling, grammar and conciseness. The browser plugin checks over content being written in real-time and alerts for everything from spelling mistakes to tone errors and even scans content for plagiarism. - -[Tempus](https://www.tempus.com) - -- Uses AI to gather and analyze massive pools of medical and clinical data at scale. The company, with the assistance of AI, provides precision medicine that personalizes and optimizes treatments to each individual’s specific health needs; relying on everything from genetic makeup to past medical history to diagnose and treat. Tempus is currently focusing on using AI to create breakthroughs in cancer research. - ---- - -#### Reference - -- https://www.simplilearn.com/top-technology-trends-and-jobs-article -- https://www.information-age.com/industries-ai-impact-123475559/ diff --git a/notes/Technology Trend/Why we need to follow technology trend.md b/notes/Technology Trend/Why we need to follow technology trend.md deleted file mode 100644 index e1392f507..000000000 --- a/notes/Technology Trend/Why we need to follow technology trend.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - technology -redirect: - - /4N7EgQ ---- - -## Why it is important to keep up with changing technology trends - -- Remain Relevant -- Stay Competitive -- Retain Customers -- Greater Opportunities - -## Why Is It Important to Keep Abreast of Technology Trends When Running an E-Business? - -- Technology Trends -- Product Anticipation -- Web-Based Applications -- Emerging Online Venues -- Trends Define Business - -## Why is technology trend awareness important - -Trend awareness as a,in the market or industry.&text=Ideal technology trend awareness skills,technology adoption or surpass it. It helps enhancing your visibility to the customers - ---- - -#### Reference - -- https://www.bigfin.com/blog/why-it-is-important-to-keep-up-with-changing-technology-trends/ -- https://smallbusiness.chron.com/target-market-technology-have-13818.html -- https://www.cleverism.com/skills-and-tools/technology-trend-awareness/#:~:text=Technology diff --git a/notes/Throttling Pattern.md b/notes/Throttling Pattern.md deleted file mode 100644 index 6a8705621..000000000 --- a/notes/Throttling Pattern.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - pattern - - system-design - - throttling -redirect: - - /mCW16g ---- - -Control the consumption of resources used by an instance of an application, an individual tenant, or an entire service. - -### Context - -- The load on a cloud application typically varies over time based on the number of active users or the types of activities they are performing. -- Sometime, huge number of request exceeded system's capacity, make system have poor performance or event failure. - -### Solution - -If it's not suitable for autoscaling, an alternative strategy to autoscaling is to allow applications to use resources only up to a limit, and then throttle them when this limit is reached. - -- Rejecting requests from an individual user who's already accessed system APIs more than n times per second over a given period of time. -- Disabling or degrading the functionality of selected nonessential services so that essential services can run unimpeded with sufficient resources. For example, if the application is streaming video output, it could switch to a lower resolution. -- Using load leveling to smooth the volume of activity (Would show detail about this in Queue-based Load Leveling pattern) + Priority Queue pattern -- Deferring operations being performed on behalf of lower priority applications or tenants. - -### Consideration - -- Impact whole system, should consider in early stage of system design. -- Throttling must be performed quickly. -- If a service needs to temporarily deny a user request, it should return a specific error code so the client application understands that the reason -- Throttling can be used as a temporary measure while a system auto-scales. -- If your system still can not operation after throttling, we should consider scaling it up for better capacity - -### When to use - -- ensure that a system continues to meet service level agreements -- prevent a single tenant from monopolizing the resources provided by an application -- handle bursts in activity -- Cost optimization - -![[Pattern_Throttling.png]] - ---- - -#### Reference - -- https://docs.microsoft.com/en-us/azure/architecture/patterns/throttling -- https://docs.firstdecode.com/microservices-architecture-style/design-patterns-for-microservices/throttling-pattern/ diff --git a/notes/Transaction patterns in microservice.md b/notes/Transaction patterns in microservice.md deleted file mode 100644 index bd7cb79a6..000000000 --- a/notes/Transaction patterns in microservice.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - microservice - - pattern -redirect: - - /8B3g3Q ---- - -### Two-phase commit pattern - -- Global transaction context -- Prepare phrase, lock resources -> commit phase -- Abort if error - -**Pros** - -- Atomic transaction, read-write isolation - -**Cons** - -- Synchronous, delay during calling services, blocking resources -> bottleneck, deadlock - -**Q&A** How do we handling errors during commit phrase -> retry on commit error?!, global transaction context tracking and calling delete all committed objects?! - -### Saga pattern - -- Local transaction, communicate through event bus -- Send rollback event to rollback changes (compensation transaction) - -**Pros** - -- Asynchronous, local transaction so there is no lock on any object/row - -**Cons** - -- Difficult to bug and maintain event messages as system grows -- No read isolation: committed data of a service might be gone after a second due to compensation transaction - ---- - -#### Reference - -https://developers.redhat.com/blog/2018/10/01/patterns-for-distributed-transactions-within-a-microservices-architecture diff --git a/notes/Why does Git use a cryptographic hash function?.md b/notes/Why does Git use a cryptographic hash function?.md deleted file mode 100644 index 8df367bc4..000000000 --- a/notes/Why does Git use a cryptographic hash function?.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - git - - practice -redirect: - - /tttXDg ---- - -Why does Git use [SHA-1](http://en.wikipedia.org/wiki/SHA-1), a cryptographic hash function, instead of a faster non-cryptographic hash function? - -## Related question - -Stack Overflow question _[Why does Git use SHA-1 as version numbers?](https://stackoverflow.com/questions/11233591)_ asks why Git uses SHA-1 as opposed to sequential numbers for commits. - -TLDR; - -- From 2005 up to 2018/Git 2.18: [SHA-1](https://en.wikipedia.org/wiki/SHA-1) (see below) -- [2019, will switch at some point](https://stackoverflow.com/a/47838703/6309) to [SHA-256](https://en.wikipedia.org/wiki/SHA-2) - ---- - -You can check that from [Linus Torvalds himself, when he presented Git to Google back in 2007](http://www.youtube.com/watch?v=4XpnKHJAok8&t=56m16s): (emphasis mine) - -> We check checksums that is considered cryptographically secure. Nobody has been able to break SHA-1, but the point is, **SHA-1 as far as git is concerned, isn't even a security feature. It's purely a consistency check**. The security parts are elsewhere. A lot of people assume since git uses SHA-1 and SHA-1 is used for cryptographically secure stuff, they think that it's a huge security feature. It has nothing at all to do with security, it's just the best hash you can get. -> -> **Having a good hash is good for being able to trust your data**, it happens to have some other good features, too, it means when we hash objects, we know the hash is well distributed and we do not have to worry about certain distribution issues. -> -> Internally it means from the implementation standpoint, we can trust that the hash is so good that we can use hashing algorithms and know there are no bad cases. -> -> So there are some reasons to like the cryptographic side too, but it's really about the ability to trust your data. I guarantee you, if you put your data in git, you can trust the fact that five years later, after it is converted from your harddisc to DVD to whatever new technology and you copied it along, **five years later you can verify the data you get back out is the exact same data you put in. And that is something you really should look for in a source code management system**. - ---- - -Update Dec. 2017 with Git 2.16 (Q1 2018): this effort to support an alternative SHA is underway: see "[Why doesn't Git use more modern SHA?](https://stackoverflow.com/a/47838703/6309)". - ---- - -I mentioned in "[How would git handle a SHA-1 collision on a blob?](https://stackoverflow.com/a/9392525/6309)" that you _could_ engineer a commit with a particular SHA1 _prefix_ (still an extremely costly endeavor). But the point remains, as [Eric Sink](http://ericsink.com/) mentions in "[Git: Cryptographic Hashes](http://ericsink.com/vcbe/html/cryptographic_hashes.html)" ([Version Control by Example (2011) book](http://ericsink.com/vcbe/html/index.html): - -> It is rather important that the DVCS never encounter two different pieces of data which have the same digest. Fortunately, good cryptographic hash functions are designed to make such collisions extremely unlikely. - -It is harder to find [good non-cryptographic hash](http://en.wikipedia.org/wiki/List_of_hash_functions) with low collision rate, unless you consider research like "[Finding State-of-the-Art Non-cryptographic Hashes with Genetic Programming](http://e-archivo.uc3m.es/bitstream/handle/10016/3987/finding_LNCS_2006_ps.pdf?sequence=1)". - -You can also read "[Consider use of non-cryptographic hash algorithm for hashing speed-up](https://github.com/markfasheh/duperemove/issues/24)", which mentions for instance "[**xxhash**](https://code.google.com/p/xxhash/)", an extremely fast non-cryptographic Hash algorithm, working at speeds close to RAM limits. - ---- - -Discussions around changing the hash in Git are not new: - -- Either [to optimize it (August 2009)](http://www.mail-archive.com/bug-coreutils@gnu.org/msg17297.html), but you have to take license issue: - -(Linus Torvalds) - -> There's not really anything _remaining_ of the mozilla code, but hey, I started from it. In retrospect I probably should have started from the PPC asm code that already did the blocking sanely - but that's a "20/20 hindsight" kind of thing. -> -> Plus hey, the mozilla code being a horrid pile of crud was why I was so convinced that I could improve on things. So that's a kind of source for it, even if it's more about the motivational side than any actual remaining code ;) - -And you need to be careful about [how to measure the actual optimization gain](http://www.mail-archive.com/bug-coreutils@gnu.org/msg17305.html) - -(Linus Torvalds) - -> I pretty much can guarantee you that it improves things only because it makes gcc generate crap code, which then hides some of the P4 issues. - -- Or to [change it altogether (January 2010)](http://lwn.net/Articles/370907/) (for instance to SHA-3, but that would apply to any other hash): - -(John Tapsell - [`johnflux`](https://github.com/johnflux)) - -> **The engineering cost for upgrading git from SHA-1 to a new algorithm is much higher**. I'm not sure how it can be done well. -> -> First of all we probably need to deploy a version of git (let's call it version 2 for this conversation) which allows there to be a slot for a new hash value even though it doesn't read or use that space -- it just uses the SHA-1 hash value which is in the other slot. -> -> That way once we _eventually_ deploy yet a newer version of git, let's call it version 3, which produces SHA-3 hashes in addition to SHA-1 hashes, people using git version 2 will be able to continue to inter-operate. (Although, per this discussion, they may be vulnerable and people who rely on their SHA-1-only patches may be vulnerable.) - -In short, switching to _any_ hash is not easy. - ---- - -Update February 2017: yes, it is in theory possible to compute a colliding SHA1: [**shattered.io**](https://shattered.io/) - -> How is GIT affected? -> -> GIT strongly relies on SHA-1 for the identification and integrity checking of all file objects and commits. It is essentially possible to create two GIT repositories with the same head commit hash and different contents, say a benign source code and a backdoored one. An attacker could potentially selectively serve either repository to targeted users. This will require attackers to compute their own collision. - -But: - -> This attack required over 9,223,372,036,854,775,808 SHA1 computations. This took the equivalent processing power as 6,500 years of single-CPU computations and 110 years of single-GPU computations. - -So let's not panic just yet. See more at "[**How would Git handle a SHA-1 collision on a blob?**](https://stackoverflow.com/a/42450327/6309)". diff --git a/notes/XML and JSON.md b/notes/XML and JSON.md deleted file mode 100644 index 7c0ff709b..000000000 --- a/notes/XML and JSON.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: null -description: null -date: null -tag: data -redirect: - - /H3HdeQ ---- - -### XML - -- Markup language, to store and process data into computer readable-format -- Not a programming language since it does not perform algorithms or computation -- Process and format objects due to the tag structures -- Complex and hard to read by human -- XML is older, this markup language was able to define a set of rules and structure to make the interchange of data universal and further produce documents that are both human and computer-readable. -- XML plays a significant role in data storage, - -### JSON - -- Data format in JS applications -- Receives data from a web server and transmits it to a web page -- Simplify data transfer due to no processing/ computation -- Empty tags/data -- More modern approach to the same purpose as XML. However, it's favored for data delivery between browsers and servers due to the lighter and faster files it produces -- JSON is the fastest and easiest way to fulfill the data structuring and exchanging mechanism. -- surpass in performance - -### The comparison - -First, as previously mentioned, while XML is a markup language, JSON, on the other hand, is a data format. One of the most significant advantages of using JSON is that the file size is smaller; thus, transferring data is faster than XML. Moreover, since JSON is compact and very easy to read, the files look cleaner and more organized without empty tags and data. The simplicity of its structure and minimal syntax makes JSON easier to be used and read by humans. Contrarily, XML is often characterized for its complexity and old-fashioned standard due to the tag structure that makes files bigger and harder to read. - -However, JSON vs. XML is not entirely a fair comparison. JSON is often wrongly perceived as a substitute for XML, but while JSON is a great choice to make simple data transfers, it does not perform any processing or computation. XML might be "old" and complex, but its complexity is what enables this language to not only transfer data but also to process and format objects and documents. - ---- - -#### Reference - -- https://dzone.com/articles/json-vs-xml-which-one-is-better diff --git a/notes/assets/autonomy-automation.webp b/notes/assets/autonomy-automation.webp deleted file mode 100644 index c91fc9ae2..000000000 Binary files a/notes/assets/autonomy-automation.webp and /dev/null differ diff --git a/notes/assets/dwarves-radio-talk-16-run-an-effective-performance-review_e46576a1c9314d3a36be38e50ae55763_md5.webp b/notes/assets/dwarves-radio-talk-16-run-an-effective-performance-review_e46576a1c9314d3a36be38e50ae55763_md5.webp deleted file mode 100644 index de00963f1..000000000 Binary files a/notes/assets/dwarves-radio-talk-16-run-an-effective-performance-review_e46576a1c9314d3a36be38e50ae55763_md5.webp and /dev/null differ diff --git a/notes/assets/dwarves-radio-talk-17-conduct-a-1-1-session_06395a8d9f4970db75d51feef9c89fa0_md5.webp b/notes/assets/dwarves-radio-talk-17-conduct-a-1-1-session_06395a8d9f4970db75d51feef9c89fa0_md5.webp deleted file mode 100644 index b80248384..000000000 Binary files a/notes/assets/dwarves-radio-talk-17-conduct-a-1-1-session_06395a8d9f4970db75d51feef9c89fa0_md5.webp and /dev/null differ diff --git a/notes/assets/multi-mcp-data-integration-0.png b/notes/assets/multi-mcp-data-integration-0.png deleted file mode 100644 index 88f16b94d..000000000 Binary files a/notes/assets/multi-mcp-data-integration-0.png and /dev/null differ diff --git a/notes/assets/multi-mcp-data-integration-1.png b/notes/assets/multi-mcp-data-integration-1.png deleted file mode 100644 index b925a2e78..000000000 Binary files a/notes/assets/multi-mcp-data-integration-1.png and /dev/null differ diff --git a/notes/assets/multi-mcp-data-integration-2.png b/notes/assets/multi-mcp-data-integration-2.png deleted file mode 100644 index e1177711b..000000000 Binary files a/notes/assets/multi-mcp-data-integration-2.png and /dev/null differ diff --git a/notes/assets/multi-mcp-data-integration-3.png b/notes/assets/multi-mcp-data-integration-3.png deleted file mode 100644 index 6f7516868..000000000 Binary files a/notes/assets/multi-mcp-data-integration-3.png and /dev/null differ diff --git a/notes/assets/multi-mcp-data-integration-4.png b/notes/assets/multi-mcp-data-integration-4.png deleted file mode 100644 index d92c58578..000000000 Binary files a/notes/assets/multi-mcp-data-integration-4.png and /dev/null differ diff --git a/notes/assets/recording-flow_untitled-3.webp b/notes/assets/recording-flow_untitled-3.webp deleted file mode 100644 index be6eb6ab6..000000000 Binary files a/notes/assets/recording-flow_untitled-3.webp and /dev/null differ diff --git a/notes/assets/recording-flow_untitled-4.webp b/notes/assets/recording-flow_untitled-4.webp deleted file mode 100644 index 6a84994b3..000000000 Binary files a/notes/assets/recording-flow_untitled-4.webp and /dev/null differ diff --git a/notes/assets/recording-flow_untitled-5.webp b/notes/assets/recording-flow_untitled-5.webp deleted file mode 100644 index 6736769de..000000000 Binary files a/notes/assets/recording-flow_untitled-5.webp and /dev/null differ diff --git a/notes/assets/recording-flow_untitled-6.webp b/notes/assets/recording-flow_untitled-6.webp deleted file mode 100644 index a900d0651..000000000 Binary files a/notes/assets/recording-flow_untitled-6.webp and /dev/null differ diff --git a/notes/assets/ux-benchmarking.webp b/notes/assets/ux-benchmarking.webp deleted file mode 100644 index 3081887c2..000000000 Binary files a/notes/assets/ux-benchmarking.webp and /dev/null differ diff --git a/notes/git.md b/notes/git.md deleted file mode 100644 index 083c1ba4d..000000000 --- a/notes/git.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - git - - git-flow -redirect: - - /k925EQ ---- - -### What is git? - -Git is a mature, actively maintained open source project originally developed in 2005 by Linus Torvalds. Git is software for tracking changes in any set of files, usually used for coordinating work among programmers collaboratively developing source code during software development. Git track all versions of your changes from the beginning of a project. The system manages the revision history for tracing, reviewing, continuous integration to collaborate efficiently. - -### Why are we using git? - -- Git is a revision control system for documents, we usually use this system for tracking the changes in a set of files. It helps groups of developers work collaboratively on software projects. -- Git is a working standard in software development. Almost all company or development teams are using git for daily work, interview sessions. -- All open sources are using git for development, collaboration. Collaborator or contributor must have experienced with git to contribute to the open-source project. - -#### Concept - -- `snapshot` is the content (files and folders) of a repository at some point in time -- `commit` in a git repository records a snapshot of all the (tracked) files in your directory. A commit has reference to the parent commit. -- `branch` is similar to the branch of a tree which contains the commits. When we make a branch from existing branches, we have a new branch and the existed branch is called a base branch. -- `repository` (or repo for short) contains all of the project files and the entire revision history. There are 2 types of repositories: local repository and remote repository. A local repository is storing data in a computer gives you a personal working environment. A remote repository is storing a copy of your Git repo with an online host (such as GitHub or Bitbucket) gives you a centrally located place where you can upload your changes and download changes from others, letting you collaborate more easily with other developers. - -### How to use git effectively? - -#### GitHub flow - -GitHub flow is a lightweight, branch-based workflow that supports teams and projects where deployments are made regularly. Using only one `master` branch for tracking your project. There are 4 steps in the GitHub flow: implement, create merge request & review, deploy to production, merge to the codebase: - -- When we get a new feature, we check out from `master` branch to another branch. Working on that branch and create a merge request to `master` branch. -- Discuss with your team and stakeholder about the feature for review and testing. -- Deploy the feature to production for final testing. -- Merge that branch to the codebase after the changes are verified in production. - -#### Git-flow - -- There're many members in your team, and they collaborate on the same project. Your team is building software that is explicitly versioned or needs to support multiple versions of your software in the wild, then git-flow is a good fit. Git-flow was first published and made popular by Vincent Driessen. The Git-flow Workflow defines a strict branching model designed around the project release. This provides a robust framework for managing larger projects. Git-flow focuses on stable and qualified products. -- The central repo holds two main branches with an infinite lifetime is `master` and `develop` The development model uses a variety of supporting branches to aid parallel development between team members, ease tracking of features, prepare for production releases, and to assist in quickly fixing live production problems. These are `feature branches`, `release branches`, `hotfix branches`. - -##### Develop feature flow - -- Base branch: develop -- Merge back to: develop -- Branch naming convention: `feat/*` -- Person in-charge: developers -- Objective: Implement a new feature of the software -- Developers check out from the `develop` branch a new feature branch. They make an early merge request with a full description of the working task. Developers work on this branch and merge to `develop` after being reviewed by teammates or leaders. - -##### Release flow - -- Base branch: develop -- Merge back to: develop and master branch -- Branch naming convention: `release/v*.*.*` -- Person in-charge: leader -- Objective: prepare and release a new version of the software -- The leaders check out from the `develop` branch a release branch. They should pump the version and update the release note. After preparing the release information, they merge the changes to `master` and `develop` branches. The leader creates a new tag from `master` branch. - -##### Hot-fix flow - -- Base branch: master -- Merge back to: develop and master branch -- Branch naming convention: `hotfix/*`, `bugfix/*` -- Person in-charge: developer or leader -- Objective: fix the urgent issue and release a new version of the software -- Developers check out from the `master` branch a hot-fix branch. After working on the source code, they should pump the version and update the changelog. Developers merge the change to `master` and `develop` branches. - -#### Git-flow in dwarves foundation - -Dwarves Foundation team is using the git-flow and some customization. We base on the idea about git-flow when working with the features branches. However, our team applies a different release flow. There are 4 environments for the development of live-cycle in the Dwarves Foundation team: `develop`, `testing`, `staging`, and `production`. Instead of release on `develop` and `master` branches, we release the product on tags. `v*.*.*` tag for production, `v*.*.*-rc` for staging, and `v*.*.*-alpha` for testing. - -##### Develop feature flow - -- Base branch: develop -- Merge back to: develop -- Branch naming convention: `feat/*` -- Person in-charge: developers -- Objective: Implement a new feature of the software -- Developers check out from the `develop` branch a new feature branch. They make an early merge request with a full description of the working task. Developers work on this branch and merge to `develop` after being reviewed by teammates or leaders. - -##### Release testing version - -- Base branch: develop -- Merge back to: develop -- Branch naming convention: `release/v*.*.*-alpha` -- Tag: `v*.*.*-alpha` -- Objective: prepare and release a new version for the testing. The QA team uses the testing environment to verify the features are a match with requirements. - -##### Release staging version - -- Base branch: `release/v*.*.*-alpha` -- Merge back to: develop, `release/v*.*.*-alpha` -- Branch naming convention: `release/v*.*.*-rc` -- Tag: `v*.*.*-rc` -- Objective: prepare and release a new version for the staging. The customers and our team make an acceptance test on the staging environment. The beta-testing is also deployed and released on it. - -##### Release production version - -- Base branch: `release/v*.*.*-rc` -- Merge back to: develop, `release/v*.*.*-rc`, master -- Branch naming convention: `release/v*.*.*` -- Tag: `v*.*.*` -- Objective: prepare and release a new version for the production. The end-users will be received the update after the features are verified through many protection checkpoints. - -##### Hot-fix flow - -- Base branch: `release/*` -- Merge back to: `release/*`, `develop` and `master` -- Branch naming convention: `hotfix/*`, `bugfix/*` -- Person in-charge: developer or leader -- Objective: fix the urgent issue and release a new version of the software -- Developers check out from the `release/*` branch a hot-fix branch. After working on the source code, they should pump the version and update the changelog. Developers merge the change to `release/*` branches after being reviewed. Additionally, We must merge to `master` branch when the current branch is the production release branch - -#### Practices - -- Using Github flow for a small project or medium project. -- Using Git-flow or DF modified git-flow for collaborative projects, multiple-version projects, large projects. - -#### Conclusion - -Git can sometimes be complex to get your head around. Most of us learn Git up to a point where we're happy to use it day-to-day and then stick to the few commands that we are comfortable with without trying anything too fancy. Most of the time it works out, but then everyone on us certainly meets the point when we mess up the repo with rebase or squash. We get panic and go to stackoverflow to seek out help. - -There are a number of factors that have held developers back from becoming super productive with Git. But the most common one, I believe, is that they don't have a solid mental model of how Git works. Consider software as an accumulation of changes over a period of time, speaking in terms of Git, a single change is called a commit. Each commit may connect with one or other commits to form a non-binary tree called Git history. Git provides a set of commands to interact with the tree by adding nodes, editing nodes, traveling between nodes, etc. Simple to complex, you need to distinguish: - -- Structure: commit vs branch -- Where: remote vs local -- File states: untracked vs tracked -- File status: unmodified vs modified vs staged -- Travel between nodes: checkout vs reset -- Move nodes: merge vs rebase vs cherry-pick -- Alter node(s): amend vs squash vs rebase -- View history: log vs reflog -- Reference: HEAD, FETCH_HEAD, HEAD~, HEAD^, tag -- Other: stash, clean, revert - -Overall, Git acts like a time machine. When you travel from A to B in time, you should always expect there is a button to come back to A. - ---- - -#### Reference - -- https://learngitbranching.js.org -- https://nvie.com/posts/a-successful-git-branching-model/ -- https://danielkummer.github.io/git-flow-cheatsheet/ -- https://github.com/k88hudson/git-flight-rules diff --git a/notes/hr/Dermatoglyphics/assets/76653027_121106256000537_5175489514994401280_n.png b/notes/hr/Dermatoglyphics/assets/76653027_121106256000537_5175489514994401280_n.png deleted file mode 100644 index 6e4fe81e4..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/76653027_121106256000537_5175489514994401280_n.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Loop.png b/notes/hr/Dermatoglyphics/assets/Loop.png deleted file mode 100644 index e51b3735f..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Loop.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/PlainArch.png b/notes/hr/Dermatoglyphics/assets/PlainArch.png deleted file mode 100644 index 5338c2d37..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/PlainArch.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36 1.png b/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36 1.png deleted file mode 100644 index ae116de16..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36 1.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36.png b/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36.png deleted file mode 100644 index ae116de16..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-06-30 at 16.41.36.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-07 at 16.19.00.png b/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-07 at 16.19.00.png deleted file mode 100644 index e1384e519..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-07 at 16.19.00.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-15 at 12.06.17.png b/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-15 at 12.06.17.png deleted file mode 100644 index e26d684c5..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-07-15 at 12.06.17.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-09-06 at 12.52.42.png b/notes/hr/Dermatoglyphics/assets/Screenshot 2021-09-06 at 12.52.42.png deleted file mode 100644 index a016ab700..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Screenshot 2021-09-06 at 12.52.42.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/Whorl.png b/notes/hr/Dermatoglyphics/assets/Whorl.png deleted file mode 100644 index 0436baf92..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/Whorl.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/brain-finger-hipothesis-1.jpeg b/notes/hr/Dermatoglyphics/assets/brain-finger-hipothesis-1.jpeg deleted file mode 100644 index 5f37cbc22..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/brain-finger-hipothesis-1.jpeg and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/chung-van-tay-Whorl.gif b/notes/hr/Dermatoglyphics/assets/chung-van-tay-Whorl.gif deleted file mode 100644 index 4390386b6..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/chung-van-tay-Whorl.gif and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/nhan dien van tay.png b/notes/hr/Dermatoglyphics/assets/nhan dien van tay.png deleted file mode 100644 index 6e4fe81e4..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/nhan dien van tay.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/van dai bang.jpeg b/notes/hr/Dermatoglyphics/assets/van dai bang.jpeg deleted file mode 100644 index ba1c8787f..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/van dai bang.jpeg and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/van nui.png b/notes/hr/Dermatoglyphics/assets/van nui.png deleted file mode 100644 index 309365af4..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/van nui.png and /dev/null differ diff --git a/notes/hr/Dermatoglyphics/assets/van nuoc.jpeg b/notes/hr/Dermatoglyphics/assets/van nuoc.jpeg deleted file mode 100644 index b18c943af..000000000 Binary files a/notes/hr/Dermatoglyphics/assets/van nuoc.jpeg and /dev/null differ diff --git a/notes/hr/mbti/(Hubspot) Apply MBTI to Sales.md b/notes/hr/mbti/(Hubspot) Apply MBTI to Sales.md deleted file mode 100644 index 16981ae78..000000000 --- a/notes/hr/mbti/(Hubspot) Apply MBTI to Sales.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - mbti - - sales -redirect: - - /y98txg ---- - -#### What personality types succeed in sales? - -1. ESFJ (The Provider/ Consul) considered the best for after sales professionals and account managers - -2. ESTP (The Explorer/Entrepreneur ) useful for short-term projects – such as new client acquisition or closing sales after others have put in the groundwork - -3. ENTJ (The Commander) independent field sales agents - -4. INTJ (The Architect) marketing, or preparing a sales case to be implemented by other colleagues. - -5. ISTP (The Virtuoso) strategy, good sales managers and team leaders - ---- - -#### Reference - -- [Hubspot](https://blog.hubspot.com/sales/manage-salespeople-myers-briggs) -- [emlteam](https://blog.emlteam.com/sales-personality-myers-briggs-test/) diff --git a/notes/hr/mbti/(Zapier) Build team with personality tests.md b/notes/hr/mbti/(Zapier) Build team with personality tests.md deleted file mode 100644 index 0a8003727..000000000 --- a/notes/hr/mbti/(Zapier) Build team with personality tests.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - mbti - - team-building -redirect: - - /CgpAkg ---- - -Here are a few ways you can use personality assessments to improve your teams and optimize for more successful projects: - -- Make personality assessments part of onboarding -- Strive for diversity -- Rotate roles based on project stages -- Know your own style - ---- - -#### Reference - -[Practices from Zapier](https://zapier.com/blog/personality-tests/) diff --git a/notes/hr/mbti/8 Cognitive Functions.md b/notes/hr/mbti/8 Cognitive Functions.md deleted file mode 100644 index f9c3aef6a..000000000 --- a/notes/hr/mbti/8 Cognitive Functions.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: null -description: null -date: null -tag: function, mbti -redirect: - - /o2xOTw ---- - -Based on [8 preferences](obsidian://open?vault=dwarves&file=brain%2FHR%2FMBTI%2F8%20Preferences) (I,E,N,S,T,F,P,J) - -- Sort by (I) and (E): outwardly and not outwardly expressed I: Ni Si Ti Fi E: Ne Se Te Fe - -- Sort by (P) and (J): how to receive information and how to process information - -P: Ni Ne Si Se J: Ti Te Fi Fe - ---- - -#### Reference - -[Functional stack theory](https://personalityjunkie.com/functional-stack-type-dynamics-theory/) [Functional stack 101](https://www.youtube.com/watch?v=9xf13VRNPp8) diff --git a/notes/hr/mbti/8 Preferences.md b/notes/hr/mbti/8 Preferences.md deleted file mode 100644 index 483290448..000000000 --- a/notes/hr/mbti/8 Preferences.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: null -description: null -date: null -tag: mbti -redirect: - - /WUPU8w ---- - -- I (introverted) >< E (extroverted) -- N (Intution) >< S (Sensing) -- T (Thinking) >< F (Feeling) -- P (Perceiving) >< J (Judging) - ---- - -- 1st place: I or E (mind) -- 2nd place: S or N (energy) -- 3rd place: T or F (nature) -- 4th place: P or J (tactics) - -2x2x2x2 = 16 personality types trong MBTI - ---- - -#### Reference - -[Typhology 101](https://personalityjunkie.com/typology-101/#anchor1) diff --git a/notes/hr/mbti/Calculate function stack.md b/notes/hr/mbti/Calculate function stack.md deleted file mode 100644 index 2cc238d42..000000000 --- a/notes/hr/mbti/Calculate function stack.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: null -description: null -date: null -tag: function, mbti -redirect: - - /zPoczQ ---- - -#### Rule - -The first letter (I or E) indicates the trend (internal - external) of the 4 functions. If it is Exxx, the set of functions will be E-I-E-I. Otherwise, if it is Ixxx, the function set will be I-E-I-E - -> Cause: type E will show their most dominant out. What outsiders can clearly see is their signature trait. Go back to type I, what outsiders see only as auxiliary. Because the signature trait they hide inside. - -The 2nd (N or S) and 3 (F or T) letters indicate what the first 2 functions are. For example, xNFx would be N & F. And xSTx would be S & T being the first 2 functions. - -The last letter (P or J) indicates that the first extrovert function is perceiving or judging. If it is xxxP, then 1 of the first 2 functions will be Ne or Se. If it is xxxJ, then one of the two functions will be Te or Fe. - -We have the opposite functions dominant <> interior ; auxiliary <>tertiary - -> has four opposing function pairs: Si <> Ne Ni <> Se Ti <> Fe Fi <> Te - -A stack is made up of two function pairs. Depending on the position, it manifests differently (strong or weak). Each such combo will form an inner world of a PT. - -Ex: ENFJ - -1 Fe 2 Ni 3 Se 4 Ti - ---- - -#### Reference - -[Functional stack theory](https://personalityjunkie.com/functional-stack-type-dynamics-theory/) [Functional stack 101](https://www.youtube.com/watch?v=9xf13VRNPp8) diff --git a/notes/hr/mbti/Cognitive Function Stack.md b/notes/hr/mbti/Cognitive Function Stack.md deleted file mode 100644 index d7fdea97c..000000000 --- a/notes/hr/mbti/Cognitive Function Stack.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: null -description: null -date: null -tag: function, mbti -redirect: - - /UsZJ6A ---- - -Each PT has 4 [cognitive functions](obsidian://open?vault=dwarves&file=brain%2FHR%2FMBTI%2F8%20Cognitive%20Functions) that called cognitive fuctions stack. t is arranged in order of 4 layers from strong > weak: - -- Dominant -- Auxiliary -- Tertiary -- Inferior - -![[Screenshot 2021-07-15 at 12.06.17.png]] - -Each personality type has a certain stack and does not change. The first two are the most obvious, what outsiders see (dominant and auxiliary). In contrast, tertiary and inferior represent our weakness and blind spot. What we need to improve to achieve satisfaction and balance in life. - -Ex: INFP's cognitive function stack is Fi Ne Si Te - ---- - -#### Reference - -[Functional stack theory](https://personalityjunkie.com/functional-stack-type-dynamics-theory/) [Functional stack 101](https://www.youtube.com/watch?v=9xf13VRNPp8) diff --git a/notes/hr/mbti/How to work with all the MBTI types.md b/notes/hr/mbti/How to work with all the MBTI types.md deleted file mode 100644 index a4eef94a1..000000000 --- a/notes/hr/mbti/How to work with all the MBTI types.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - mbti - - team-building -redirect: - - /qP3rlQ ---- - -The best practices to work with all MBTI types, ref from Atlassian - -#### ISTJ: Logistician - -- Don’t need much management or checking up on, just email them a to-do list at the start of the week, and you can trust them to quietly get on with it. -- The perfect type to organize and streamline any shared space online, from Dropbox to Google Docs. - -#### INFJ: Advocate - -- It would be a major asset to have them sit in on virtual interviews. -- Just don’t put them center stage in any Zoom meetings if you can avoid it: they don’t thrive on attention, and work far better behind the scenes. - -#### INTJ: Architect - -- More comfortable communicating by text -- Extremely deadline-focused, but there’s also a danger they can rush to hasty decisions, particularly without colleagues nearby to check their impulses. -- Need to be reminded to stop for a second, take their time, and let ideas germinate, rather than just rushing straight at them. - -#### ISFJ: Defender - -- Display incredible attention to detail, so they’re great for checking over others’ work, editing shared documents or looking over pitches and proposals at the final stage -- Very good at following rules and inspiring others to do the same - -#### ISTP: Virtuoso - -- Tend to excel at troubleshooting, so in a remote work environment, they can be a major tech asset. -- Very good at test driving new tools and navigating software, but they also lose focus easily so they do need to be managed. - -#### SFP: Adventurer - -- Can burn out quite easily -- Their energy is an asset, but it sometimes needs to be harnessed and directed in the right direction by others. - -#### INFP: Mediator - -- Tends to have very deep-seated values, which can cause problems because frustrations can stew when they’re offended, -- Managers need to encourage them to get any concerns out into the open. Otherwise, the key to getting the best out of this group is to encourage and reinforce meaning in their work. - -#### INTP: Logician - -- Really needs to be given the freedom to do things in an original way, and to be *listened* to, because they come up with the smartest solutions -- Their weak spot is that they often neglect to share decisions and solutions. If there’s an INTP on your team, encourage them to use shared documents and software as much as possible. - -#### ESTP: Entrepreneur - -- Keep things fun and keep them moving fast. This personality type is classically impatient, so give them a day’s worth of tasks in a project tool rather than any long-term targets. -- Very good at firefighting, because the thrill of the moment is exciting to them. -- Great at thinking outside the box. - -#### ESFP: Entertainer - -- Get them involved in videos, voiceovers, podcasts, or any project that involves creative performance. -- Superb in remote pitches, as they’ll bring a persuasive energy which could otherwise be lacking via computer screen. - -#### ENFP: Campaigner - -- A major asset in brainstorming sessions and any big picture thinking. -- Not the best starter-finishers, so deadlines can be an issue. -- They don’t always see messages or respond quickly enough to colleagues. As a result, they often need gently managing in order to realize their high creative value. - -#### ENTP: Debater - -- Very good at debating. This group tends to be great on new ideas and products, as well as bigger discussions about how to move the business forward -- Adept at impressing clients and pitching for new business. You just might need to remind them to mute themselves occasionally, because, if unchecked, they may dominate conversation. - -#### ESTJ: Executive - -- Loves to organize themselves, other people, and the world around them, which can be an asset but can also come across as bossy and aggressive – particularly when they’re dishing out instructions without any face-to-face contact -- Need to be reminded to be tactful with others, particularly in an environment where they’re primarily communicating via email or messaging apps, leaving their sentences open to greater interpretation. - -#### ESFJ: Consul - -- Make the best project managers, because people love working for them -- They’re organized, as well as thoughtful, so are ideal for bringing projects together on time. They’d also be a strong choice for remote onboarding new starters. - -#### ENFJ: Protagonist - -- People-focused diplomats, this group tends to forget their own needs in favor of the greater good, and that can sometimes be detrimental – not just in terms of burnout, but also when completing their own tasks. -- It’s always a good idea to have Protagonists lead group discussions, even if they’re not in a leadership role, because they excel at it. They should be your go-to Zoom meeting host, and at the heart of any situation involving discussion, consensus, and the bringing together of people and ideas. - -#### ENTJ: Commander - -- Likes big pictures and big decisions -- They don’t always see the finer details when implementing plans, that puts them at greater risk of pushing through decisions without properly taking in the views of others. -- Need to remember the necessary balance between directing and consulting. - ---- - -#### Reference - -[How to work with all the Myers-Briggs personality types](https://www.atlassian.com/blog/leadership/every-myers-briggs-personality-type) - -[Can personality tests help your team?](https://www.youtube.com/watch?v=pxlX7_ie49Q&t=66s) diff --git a/notes/hr/mbti/Introverted Intuition - Ni.md b/notes/hr/mbti/Introverted Intuition - Ni.md deleted file mode 100644 index eadffcfd0..000000000 --- a/notes/hr/mbti/Introverted Intuition - Ni.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: null -description: null -date: null -tag: function, mbti -redirect: - - /mF76bA ---- - -Book: My True Type, page 58 - 69 - -- Primary roles: To collect conscious and subconscious information, then synthesize it to produce impressions, insights, answers, and theories; to see deep causes, patterns, and laws underlying sense data -- Most prominent in: INTJs & INFJs; also prominent in ENTJs & ENFJs -- Associated with: Deep perceptiveness and insight, deep “knowing,” theoretical genius, powers of analysis and focus, advising and troubleshooting -- Famous INJs: Plato, Jesus, Nietzsche, Jung, Steven Hawking - -> Ni is the rarest of the eight types of functions, considered to have "signals of the universe". They are skilled at seeing the “dark skies before the storm.” - -> Ni likes to learn the "iceberg" of the problem - ---- - -#### Reference - -Book: My True Type Clarifying Your Personality Type, Preferences & Functions Copyright © 2014 A.J. Drenth diff --git a/notes/hr/mbti/OCEAN model.md b/notes/hr/mbti/OCEAN model.md deleted file mode 100644 index 9f2596179..000000000 --- a/notes/hr/mbti/OCEAN model.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: null -description: null -date: null -tag: mbti -redirect: - - /xJLwDg ---- - -The test on 16personalities is also combined with Five Big personality traits (OCEAN model), so the results are usually 5 characters. - -- Openness > imagination, insight (do you like new things, do you like abstract concepts, do you have many interests) -- Conscientiousness > attitude when doing tasks (do you prepare in advance/plan, details or not, often late to deadline) -- Extraversion -- Agreeableness (have sympathy, concern, like to help others) -- Neuroticism (control mood, deal with stress ntn) - -#### Note: - -- A person's Traits can change with age, and are hereditary -- Behavious = person's underlying personality + situational variables - ---- - -#### Reference - -[The big five personality dimensions](https://www.verywellmind.com/the-big-five-personality-dimensions-2795422#:~:text=The%20five%20broad%20personality%20traits,how%20many%20personality%20traits%20exist) [Wiki](https://en.wikipedia.org/wiki/Big_Five_personality_traits) diff --git a/notes/hr/mbti/Strategies for Accurate Typing.md b/notes/hr/mbti/Strategies for Accurate Typing.md deleted file mode 100644 index 9ebcc258e..000000000 --- a/notes/hr/mbti/Strategies for Accurate Typing.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: null -description: null -date: null -tag: mbti -redirect: - - /pEEavQ ---- - -Book: My True Type, page 18 - 22 - -#### Core: - -- understand the framework (function stack) -- Test -- Self-reflection -- Discuss with others - -#### Additional: - -- Childhood memories -- Which Type(s) are You Least Like? -- Which Function(s) Do You Use the Least? -- Enantiodromia: ám chỉ việc lật một thứ gì đó sang chiều ngược lại. -- Put Inconsistencies in Perspective -- Avoid Over-Focusing on Particulars - ---- - -#### Reference - -Book: My True Type Clarifying Your Personality Type, Preferences & Functions Copyright © 2014 A.J. Drenth diff --git a/notes/hr/mbti/Team building based on MBTI.md b/notes/hr/mbti/Team building based on MBTI.md deleted file mode 100644 index e4a34a641..000000000 --- a/notes/hr/mbti/Team building based on MBTI.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - mbti - - team-building -redirect: - - /4IUC5g ---- - -To use the idea of temperament most effectively, it is important to understand that the** four temperaments** are not simply arbitrary collections of characteristics, but spring from an interaction of the two basic dimensions of human behavior: **our communication** and**our action**, our words and our deeds, or, simply, what we say and what we do. - -- Communication: Concrete (talk about reality) and Abstract (talk about ideas) -- Action: Cooperative (Do what's right) and Utilitalian (Do what works) - -Psychologist David Keirsey introduced a new perspective on the Myers-Briggs types by grouping the types into four temperaments. He named these temperaments as follows: - - **Guardians (Sensation + Judgment)** - -- Excel at logistics -- Tend to be reliable, conventional, and vigilant -- Value duty, commitment, and established hierarchies -- Seek service, respect, and a sense of community - - **Artisans (Sensation + Perception)** - -- Excel at tactics -- Tend to be cheerful, spontaneous, and resourceful -- Value artistry, improvisation, and results -- Seek mastery, fun, and a competitive advantage - - **Idealists (Intuition + Feeling)** - -- Excel at diplomacy -- Tend to be kind, encouraging, and impassioned -- Value empathy, integrity, and self-expression -- Seek understanding, spiritual knowledge, and personal growth - - **Rationals (Intuition + Thinking)** - -- Excel at strategy -- Tend to be calm, practical, and skeptical -- Value technical knowledge, inventiveness, and success -- Seek expertise, autonomy, and self-sufficiency - ---- - -#### Reference - -[Four temperaments overview](https://keirsey.com/temperament-overview/) [Building Effective Teams Using the Myers-Briggs Personality Types](http://www.cmcgc.com/media/handouts/300502/0140-Wenger.pdf) diff --git a/notes/hr/mbti/Why Do We Need Cognitive Functions?.md b/notes/hr/mbti/Why Do We Need Cognitive Functions?.md deleted file mode 100644 index 8bf4c5297..000000000 --- a/notes/hr/mbti/Why Do We Need Cognitive Functions?.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: null -description: null -date: null -tag: function, mbti -redirect: - - /HzUqxA ---- - -Personality type (PT) only shows the surface of our pesonality. eg: mine is INFP, if read full description on 16personalities, it will say that your characteristics are: - -- I: introverted -- N: institution -- F: feeling -- P: preceiving - -But if we go deeper, what determines the characteristics of the personality type and why is it so? And how does it have more features and trends? That's when I learned about cognitive functions.. - ---- - -#### Reference - -[Functional stack theory](https://personalityjunkie.com/functional-stack-type-dynamics-theory/) [Functional stack 101](https://www.youtube.com/watch?v=9xf13VRNPp8) diff --git a/notes/hr/mbti/history.md b/notes/hr/mbti/history.md deleted file mode 100644 index 0a21e02b0..000000000 --- a/notes/hr/mbti/history.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: null -description: null -date: null -tag: mbti -redirect: - - /HjrrCA ---- - -MBTI (Myers-Briggs Type Indicator) > 1920s according to the theory of psychologist Carl Jung and many other frameworks in the following decades. The most important of which is the Five Big Personalities Traits. - -Type <> Trait - -Type: which side is > 50% then assign 1 type to that type Trait: based on Degree of each personality type, no classification. - ---- - -#### Reference - -[Our Theory](https://www.16personalities.com/articles/our-theory) diff --git a/notes/lstm.md b/notes/lstm.md deleted file mode 100644 index 4dbab05a4..000000000 --- a/notes/lstm.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: LSTM -description: Long Short-Term Memory (LSTM) networks are a crucial advancement for sequential data, addressing the exploding and vanishing gradient problems in traditional RNNs. This article explains how LSTMs use a clever architecture with a cell state and a hidden state, along with three gates—forget, input, and output—to manage information flow effectively. -date: 2025-07-18 -authors: - - tieubao -tags: - - AI - - LSTM - - RNN ---- - -Long short-term memory (LSTM) networks represent a significant leap forward for anyone working with sequential data. If you have ever tried to train a basic recurrent neural network (RNN) and found it struggled to learn patterns over longer sequences, you are not alone. The primary challenge? Exploding and vanishing gradients. These issues make it tough for standard RNNs to recall events from more than a few steps ago. - -LSTM changes the game by introducing a clever architecture that separates long-term memory from short-term memory. Instead of relying on a single feedback loop, LSTM networks use two distinct paths. One path, the cell state, is designed to hold onto information for the long haul. The other, the hidden state, manages details that only matter in the moment. This separation allows LSTMs to remember important events from much earlier in a sequence. Think of it as having both a notebook for key ideas and a sticky note for quick reminders. - -At the heart of every LSTM unit are three gates: - -- **Forget gate:** This gate decides what information to keep and what to discard from long-term memory. If something is no longer relevant, it gets filtered out. -- **Input gate:** This determines how much new information should be added to the long-term memory. It is where the network learns what is worth remembering. -- **Output gate:** This controls what information gets passed along as the short-term memory, influencing the network’s immediate output. - -These gates utilize two activation functions: sigmoid (which outputs values between 0 and 1, acting like a probability or percentage) and tanh (which outputs values between -1 and 1, allowing the network to represent a range of information). By combining these, LSTM networks can fine-tune what they remember, update, or ignore at every step. - -One of the best aspects of LSTM networks is their flexibility. You can unroll them over sequences of any length, and they will use the same weights and biases each time. This means you do not have to redesign your network for every new dataset. LSTM adapts to the data, not the other way around. - -If you are working with time series, language data, or any sequence where context from earlier steps matters, LSTM should be on your shortlist. It is a practical, well-crafted approach that helps your models remember what actually matters, without getting tripped up by technical limitations. - -Sources: - -- -- diff --git a/topics/agentic/ai-agent-explained.md b/topics/agentic/ai-agent-explained.md deleted file mode 100644 index 4de53e7b4..000000000 --- a/topics/agentic/ai-agent-explained.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: AI agent explained -description: Most people confuse LLMs, workflows, and agents, but understanding these three levels unlocks the real potential of AI automation. The key difference lies in who makes the decisions. -date: 2025-06-23 -authors: - - tieubao -tags: - - ai - - agents - - automation - - llm - - workflow ---- - -## The three levels that define modern AI - -**AI isn't one technology, it's three distinct levels of capability.** Most people lump everything under "AI" without understanding the crucial differences between large language models, AI workflows, and true AI agents. Each level represents a fundamental shift in who controls the decision-making process. - -Understanding these levels isn't just academic. It determines whether you're using AI as a passive tool, an automated assistant, or an autonomous problem-solver. The distinction shapes what's possible and what you should expect from different AI implementations. - -```mermaid -%%{init: {'flowchart': {'curve': 'stepBefore'}}}%% -graph TD - - subgraph Level3 ["🤖 Level 3: AI agents"] - A3["Autonomous decision making"] - B3["Self-judgment & iteration"] - C3["Non-deterministic paths"] - D3["Goal-oriented behavior"] - end - - subgraph Level2 ["⚙️ Level 2: AI workflows"] - A2["Predefined steps"] - B2["Human decision making"] - C2["Deterministic process"] - D2["Multi-step automation"] - end - - subgraph Level1 ["📝 Level 1: Large language models"] - A1["Text generation"] - B1["Passive response"] - C1["Non-deterministic output"] - D1["Human controlled"] - end - - Level1 -.-> Level2 - Level2 -.-> Level3 - - classDef level1 fill:#e1f5fe,stroke:#0277bd,stroke-width:2px - classDef level2 fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef level3 fill:#e8f5e8,stroke:#2e7d32,stroke-width:2px - - class A1,B1,C1,D1 level1 - class A2,B2,C2,D2 level2 - class A3,B3,C3,D3 level3 -``` - -## Level one: large language models - -LLMs like ChatGPT, Gemini, and Claude represent the foundation layer. They excel at generating and editing text based on your prompts, but they operate within strict boundaries. - -**LLMs are fundamentally passive.** They respond to what you ask but cannot take independent action. They lack access to your personal data, external systems, or real-time information unless you explicitly provide it. Think of them as highly sophisticated text processors that wait for your instructions. - -**LLMs are non-deterministic by nature.** Ask the same question twice and you'll likely get different responses due to their probabilistic text generation. This limitation isn't a flaw, it's by design. LLMs provide controlled interactions where you maintain complete oversight of every exchange. - -## Level two: AI workflows - -AI workflows combine LLMs with predefined steps that humans design. These systems can fetch data from calendars, check weather services, or pull information from databases before generating responses. - -**Workflows automate multi-step processes but follow fixed paths.** They're perfect for predictable, repeatable tasks where you know the sequence of actions needed. Retrieval augmented generation (RAG) is simply a workflow where AI looks up information before answering your question. - -**Workflows are largely deterministic.** Given the same inputs, they follow the same predefined steps and produce consistent outputs. The human remains the decision maker in workflows. You design the logic, set the parameters, and adjust the process when needed. The AI executes your predetermined steps with impressive efficiency. - -## Level three: AI agents - -[**AI agents are truly autonomous.**](https://memo.d.foundation/arc/on-agent/) The LLM becomes the decision maker, reasoning about the best way to achieve your goal and taking actions using available tools. This represents the fundamental shift from human decision-making to AI decision-making. - -**Agents can judge their own work and act on that judgment.** They iterate, critique their own outputs, and improve results without constant human intervention. This self-evaluation capability sets them apart from simple automation. They use frameworks like ReAct (reasoning and acting) to plan their approach, execute actions, evaluate results, and adjust their strategy based on what they learn. - -**Agents are inherently non-deterministic.** Even with the same goal, they might take different paths, make different decisions, and arrive at varied solutions based on their reasoning process. This unpredictability is actually a feature, not a bug, as it enables creative problem-solving and adaptation. - -Consider this real-world example: an AI vision agent searches video clips for a skier by reasoning what a skier looks like, acting to find relevant footage, and returning results. No human tagged the videos or defined the search criteria. The agent figured it out autonomously, judged the quality of its findings, and refined its approach until it achieved the goal. - -## The one massive change that creates agents - -**Replace the human decision maker in your workflow with an LLM that can reason and act.** This single shift transforms passive automation into adaptive, goal-driven behavior. - -If you're still making the key decisions about what happens next, you're working with workflows, not agents. True agents take responsibility for determining the best path toward your objective. - -## Practical implementation guide - -**If you want to move from simple automation to true AI agents, focus on shifting decision making from humans to the AI.** This is the fundamental transition that unlocks autonomous behavior. - -**Start with simple LLMs for text tasks.** Use them for writing, editing, and content generation where you maintain full control over the process. - -**Add workflows for automation.** When you have predictable, multi-step tasks that benefit from external data integration, workflows provide reliable automation. - -**Graduate to agents for adaptive solutions.** Deploy agents when you need goal-oriented behavior that can adapt to changing conditions without your constant oversight. - -The key question for any AI implementation: who is making the decisions? If it's still you, you haven't reached the agent level yet. Let agents handle the trial and error so you can focus on higher-level strategic thinking. - -**The real value of agents lies in their autonomous ability to adapt, iterate, and improve.** True AI agents don't just automate your existing process. They reason about better ways to achieve your goals, judge the quality of their own work, and adapt their approach based on what they learn without requiring your oversight. diff --git a/topics/agentic/assets/the-new-problem.png b/topics/agentic/assets/the-new-problem.png deleted file mode 100644 index df18f420d..000000000 Binary files a/topics/agentic/assets/the-new-problem.png and /dev/null differ diff --git a/topics/ai/assets/adapters.webp b/topics/ai/assets/adapters.webp deleted file mode 100644 index 8b5440a36..000000000 Binary files a/topics/ai/assets/adapters.webp and /dev/null differ diff --git a/topics/ai/assets/explaining-gradient-descent-with-a-simple-analogy.webp b/topics/ai/assets/explaining-gradient-descent-with-a-simple-analogy.webp deleted file mode 100644 index 7e7b0aeab..000000000 Binary files a/topics/ai/assets/explaining-gradient-descent-with-a-simple-analogy.webp and /dev/null differ diff --git a/topics/ai/assets/explorers.webp b/topics/ai/assets/explorers.webp deleted file mode 100644 index 923b8d0d9..000000000 Binary files a/topics/ai/assets/explorers.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-1.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-1.webp deleted file mode 100644 index 12e9fe2e3..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-1.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-2.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-2.webp deleted file mode 100644 index 472987ada..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-2.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-3.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-3.webp deleted file mode 100644 index 6468c7ebd..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-3.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-4.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-4.webp deleted file mode 100644 index 3638b4441..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-4.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-5.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-5.webp deleted file mode 100644 index 6023073eb..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-5.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-6.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-6.webp deleted file mode 100644 index e6b99eab8..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-6.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-7.webp b/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-7.webp deleted file mode 100644 index a01696b3a..000000000 Binary files a/topics/ai/assets/how-we-crafted-the-ogif-summarizer-bot-to-streamline-weekly-knowledge-sharing_crafting-ogif-summarize-bot-7.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_7fbe880aa3a51713c213dc3db6043fc7_md5.webp b/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_7fbe880aa3a51713c213dc3db6043fc7_md5.webp deleted file mode 100644 index 2ee2d4327..000000000 Binary files a/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_7fbe880aa3a51713c213dc3db6043fc7_md5.webp and /dev/null differ diff --git a/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_8d4d17b65a040f22cd2482f841361f93_md5.webp b/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_8d4d17b65a040f22cd2482f841361f93_md5.webp deleted file mode 100644 index c68077dd2..000000000 Binary files a/topics/ai/assets/how-we-created-an-ai-powered-interview-system-using-openais-chatgpt_8d4d17b65a040f22cd2482f841361f93_md5.webp and /dev/null differ diff --git a/topics/ai/assets/operators.webp b/topics/ai/assets/operators.webp deleted file mode 100644 index 0aff48bb5..000000000 Binary files a/topics/ai/assets/operators.webp and /dev/null differ diff --git a/topics/ai/assets/securing-your-remote-mcp-servers-1.webp b/topics/ai/assets/securing-your-remote-mcp-servers-1.webp deleted file mode 100644 index afef45a21..000000000 Binary files a/topics/ai/assets/securing-your-remote-mcp-servers-1.webp and /dev/null differ diff --git a/topics/ai/assets/simulated-user-implementation.webp b/topics/ai/assets/simulated-user-implementation.webp deleted file mode 100644 index b0bd106f6..000000000 Binary files a/topics/ai/assets/simulated-user-implementation.webp and /dev/null differ diff --git a/topics/ai/assets/the-five-stages-of-learning.webp b/topics/ai/assets/the-five-stages-of-learning.webp deleted file mode 100644 index dc37081bc..000000000 Binary files a/topics/ai/assets/the-five-stages-of-learning.webp and /dev/null differ diff --git a/topics/ai/assets/tool-level-security-for-remote-mcp-servers.webp b/topics/ai/assets/tool-level-security-for-remote-mcp-servers.webp deleted file mode 100644 index 18f7b6b6d..000000000 Binary files a/topics/ai/assets/tool-level-security-for-remote-mcp-servers.webp and /dev/null differ diff --git a/topics/ai/explaining-gradient-descent-in-machine-learning-with-a-simple-analogy.md b/topics/ai/explaining-gradient-descent-in-machine-learning-with-a-simple-analogy.md deleted file mode 100644 index b6c1620c7..000000000 --- a/topics/ai/explaining-gradient-descent-in-machine-learning-with-a-simple-analogy.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Explaining gradient descent in machine learning with a simple analogy -description: "Gradient descent is a fundamental optimization algorithm in machine learning. It's a way for models to learn from data and improve their accuracy by gradually adjusting their internal settings. Think of it like carefully descending a hill to find the lowest point, each small step you take brings you closer to the best possible solution." -date: 2024-06-25 -authors: - - innnotruong -tags: - - ai - - machine-learning - - llm -redirect: - - /Aj87oQ ---- - -Gradient descent is a fundamental optimization algorithm in machine learning. It's a bit like finding your way down a mountain in the fog: you take small steps in the direction that seems to be going downhill the fastest. - -In machine learning, the "mountain" is a mathematical function, and the "downhill direction" helps us find the best values for our model's parameters. - -### Imagine you're sliding down a hill to find the lowest point - -**The Hill** - -Imagine you’re standing on a hill. The hill has many ups and downs, and your goal is to find the lowest point in the hill, where there’s a treasure hidden. - -**Your Steps** - -1. **Starting point**: You start somewhere on the hill. You don't know if it's the lowest point, but you’re going to find out. - -2. **Looking around**: You look around to see which direction the ground slopes downward. This tells you which way to go to get closer to the lowest point. - -3. **Taking a step**: You take a small step down the hill in that direction. - -4. **Repeat**: After taking a step, you look around again, see which way is down, and take another small step. You keep doing this until you reach the lowest point where you can’t go down any further. - -### In machine learning - -**1. The Hill**: Represents the error or how wrong the computer’s guesses are when trying to learn something (like recognizing cats and dogs). - -**2. Lowest point**: Represents the best possible way the computer can learn from the data, minimizing errors. - -**3. Steps**: Each small step you take is like the computer adjusting its guesses a little bit each time to improve its learning. - -**4. Looking around:** The computer checks how it’s doing and decides which way to adjust its guesses to make fewer mistakes. - -**5. Repeat:** The computer keeps adjusting its guesses little by little until it finds the best way to learn from the data, which is like you reaching the lowest point on the hill. - -![](assets/explaining-gradient-descent-with-a-simple-analogy.png) - -### Gradient descent in simple terms - -- **You:** The computer trying to learn. -- **Hill:** The error in guesses. -- **Lowest point:** Best learning with the least errors. -- **Steps:** Small adjustments to improve guesses. -- **Looking around:** Checking which way reduces errors the most. - -Just like you keep stepping down the hill to find the treasure, the computer keeps adjusting its guesses to learn better and make the fewest mistakes. - -### Reference - - diff --git a/topics/architecture/assets/builder-design-pattern.pdf b/topics/architecture/assets/builder-design-pattern.pdf deleted file mode 100644 index 40442872c..000000000 Binary files a/topics/architecture/assets/builder-design-pattern.pdf and /dev/null differ diff --git a/topics/architecture/assets/c4-diagrams_c4-overview.webp b/topics/architecture/assets/c4-diagrams_c4-overview.webp deleted file mode 100644 index 2e896f20e..000000000 Binary files a/topics/architecture/assets/c4-diagrams_c4-overview.webp and /dev/null differ diff --git a/topics/architecture/assets/c4-modelling.pdf b/topics/architecture/assets/c4-modelling.pdf deleted file mode 100644 index e00ca6fc2..000000000 Binary files a/topics/architecture/assets/c4-modelling.pdf and /dev/null differ diff --git a/topics/architecture/assets/command-pattern_command-comic-1-2x.webp b/topics/architecture/assets/command-pattern_command-comic-1-2x.webp deleted file mode 100644 index 89a9ac42b..000000000 Binary files a/topics/architecture/assets/command-pattern_command-comic-1-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/command-pattern_command-en-2x.webp b/topics/architecture/assets/command-pattern_command-en-2x.webp deleted file mode 100644 index e3fe36d9d..000000000 Binary files a/topics/architecture/assets/command-pattern_command-en-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/command-pattern_structure-2x.webp b/topics/architecture/assets/command-pattern_structure-2x.webp deleted file mode 100644 index 9f85671c8..000000000 Binary files a/topics/architecture/assets/command-pattern_structure-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/design-file-sharing-system_1.webp b/topics/architecture/assets/design-file-sharing-system_1.webp deleted file mode 100644 index b6b7c50d6..000000000 Binary files a/topics/architecture/assets/design-file-sharing-system_1.webp and /dev/null differ diff --git a/topics/architecture/assets/design-file-sharing-system_2.webp b/topics/architecture/assets/design-file-sharing-system_2.webp deleted file mode 100644 index ffa9182ef..000000000 Binary files a/topics/architecture/assets/design-file-sharing-system_2.webp and /dev/null differ diff --git a/topics/architecture/assets/mealy-machine_mealy_machine.webp b/topics/architecture/assets/mealy-machine_mealy_machine.webp deleted file mode 100644 index dc74cd0fc..000000000 Binary files a/topics/architecture/assets/mealy-machine_mealy_machine.webp and /dev/null differ diff --git a/topics/architecture/assets/moore-machine_moore_machine.webp b/topics/architecture/assets/moore-machine_moore_machine.webp deleted file mode 100644 index 5bc6a3112..000000000 Binary files a/topics/architecture/assets/moore-machine_moore_machine.webp and /dev/null differ diff --git a/topics/architecture/assets/observer-pattern-problem.webp b/topics/architecture/assets/observer-pattern-problem.webp deleted file mode 100644 index 0f8bc2320..000000000 Binary files a/topics/architecture/assets/observer-pattern-problem.webp and /dev/null differ diff --git a/topics/architecture/assets/observer-pattern.webp b/topics/architecture/assets/observer-pattern.webp deleted file mode 100644 index ba9376920..000000000 Binary files a/topics/architecture/assets/observer-pattern.webp and /dev/null differ diff --git a/topics/architecture/assets/overview-on-broker-pattern-in-distributed-system_e4d47aa7182bbec713b6dc4f858fb1dd_md5.webp b/topics/architecture/assets/overview-on-broker-pattern-in-distributed-system_e4d47aa7182bbec713b6dc4f858fb1dd_md5.webp deleted file mode 100644 index cf9c02c73..000000000 Binary files a/topics/architecture/assets/overview-on-broker-pattern-in-distributed-system_e4d47aa7182bbec713b6dc4f858fb1dd_md5.webp and /dev/null differ diff --git a/topics/architecture/assets/prototype-design-pattern.pdf b/topics/architecture/assets/prototype-design-pattern.pdf deleted file mode 100644 index a1384b07e..000000000 Binary files a/topics/architecture/assets/prototype-design-pattern.pdf and /dev/null differ diff --git a/topics/architecture/assets/reducers_counter_reducer_state_machine.webp b/topics/architecture/assets/reducers_counter_reducer_state_machine.webp deleted file mode 100644 index 617959534..000000000 Binary files a/topics/architecture/assets/reducers_counter_reducer_state_machine.webp and /dev/null differ diff --git a/topics/architecture/assets/reducers_flux_architecture.webp b/topics/architecture/assets/reducers_flux_architecture.webp deleted file mode 100644 index ba2c1eec6..000000000 Binary files a/topics/architecture/assets/reducers_flux_architecture.webp and /dev/null differ diff --git a/topics/architecture/assets/service-based-architecture_pasted-image-20220922153254.webp b/topics/architecture/assets/service-based-architecture_pasted-image-20220922153254.webp deleted file mode 100644 index 1b7e176a8..000000000 Binary files a/topics/architecture/assets/service-based-architecture_pasted-image-20220922153254.webp and /dev/null differ diff --git a/topics/architecture/assets/singleton-design-pattern.pdf b/topics/architecture/assets/singleton-design-pattern.pdf deleted file mode 100644 index bb8ad865d..000000000 Binary files a/topics/architecture/assets/singleton-design-pattern.pdf and /dev/null differ diff --git a/topics/architecture/assets/state-pattern_problem1-2x.webp b/topics/architecture/assets/state-pattern_problem1-2x.webp deleted file mode 100644 index 819130403..000000000 Binary files a/topics/architecture/assets/state-pattern_problem1-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/state-pattern_problem2-en-2x.webp b/topics/architecture/assets/state-pattern_problem2-en-2x.webp deleted file mode 100644 index bb7df0eb5..000000000 Binary files a/topics/architecture/assets/state-pattern_problem2-en-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/state-pattern_solution-en-2x.webp b/topics/architecture/assets/state-pattern_solution-en-2x.webp deleted file mode 100644 index f904d8a34..000000000 Binary files a/topics/architecture/assets/state-pattern_solution-en-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/state-pattern_state-en-2x.webp b/topics/architecture/assets/state-pattern_state-en-2x.webp deleted file mode 100644 index 352f352d3..000000000 Binary files a/topics/architecture/assets/state-pattern_state-en-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/state-pattern_structure-en-2x.webp b/topics/architecture/assets/state-pattern_structure-en-2x.webp deleted file mode 100644 index 4e1908ae2..000000000 Binary files a/topics/architecture/assets/state-pattern_structure-en-2x.webp and /dev/null differ diff --git a/topics/architecture/assets/strategy-design-pattern.pdf b/topics/architecture/assets/strategy-design-pattern.pdf deleted file mode 100644 index 118bdc2b8..000000000 Binary files a/topics/architecture/assets/strategy-design-pattern.pdf and /dev/null differ diff --git a/topics/architecture/assets/uml-state-machine-diagram.webp b/topics/architecture/assets/uml-state-machine-diagram.webp deleted file mode 100644 index 9b2cf53b0..000000000 Binary files a/topics/architecture/assets/uml-state-machine-diagram.webp and /dev/null differ diff --git a/topics/architecture/assets/visitor-design-pattern-1.webp b/topics/architecture/assets/visitor-design-pattern-1.webp deleted file mode 100644 index e09b33de4..000000000 Binary files a/topics/architecture/assets/visitor-design-pattern-1.webp and /dev/null differ diff --git a/topics/architecture/assets/visitor-design-pattern-2.webp b/topics/architecture/assets/visitor-design-pattern-2.webp deleted file mode 100644 index d47c36a69..000000000 Binary files a/topics/architecture/assets/visitor-design-pattern-2.webp and /dev/null differ diff --git a/topics/architecture/assets/visitor-design-pattern.pdf b/topics/architecture/assets/visitor-design-pattern.pdf deleted file mode 100644 index f6543013a..000000000 Binary files a/topics/architecture/assets/visitor-design-pattern.pdf and /dev/null differ diff --git a/topics/architecture/automata.md b/topics/architecture/automata.md deleted file mode 100644 index a24fbd6da..000000000 --- a/topics/architecture/automata.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -title: Automata -description: Things about automata that devs should know -date: 2024-10-22 -authors: - - tieubao -tags: - - automata -redirect: - - /jYvaag ---- - -### What are Finite State Automata and why should a programmer know about them? - -Formally, an FSA is a algebraic structure `F = ⟨Σ, S, s0, F, δ⟩` where `Σ` is the input alphabet, `S` is a set of states, `s0 ∈ S` is a particular start state, `F ⊆ S` is a set of accepting states, and `δ:S×Σ → S` is the state transition function. - -1/ Short answer, it is a technique that you can use to express systems with concrete states (as opposed to quantum states / probability distributions). - -Put simply, it is an effective way to represent the path(s) from a starting state to the end state(s) of the system that you care about. Using regular expressions as a fairly easy to understand example, let's look at the pattern AB+C (imagine that that plus is a superscript). I would expect to this pattern to accept strings such as "ABC", "ABBC", "ABBBC", etc. A at the start, C at the end, some number of B's in the middle (greater than or equal to one). - -If you think about it, it's almost easier to think about this in terms of a picture. Faking it with text (and that my parentheses are a loopback arc), you can see that A (on the left), is the starting state and C (on the right) is the end state on the right. - -``` - _ - ( ) -A --> B --> C -``` - -From FSAs, you can continue your journey into computational complexity by heading over to the land of Turing Machines. - -However, you can also use state machines to represent real behaviors and systems. In my world, we use them to model certain workflow of actual people working with components that are extremely intolerant of mistakes in state order. As in, "A had better happen before C or there will be a very serious problem. Make that be not possible right now." - -2/ FSA are primarily a **thinking tool**, not a programming technique. - -FSA provides a **clear, formal way** to describe and model systems with multiple states and transitions. This is useful in scenarios where systems must respond to a sequence of events. - -Being able to model systems in terms of states and transitions helps developers design clear, maintainable, and bug-free applications. - -[Source](https://stackoverflow.com/questions/364193/what-are-finite-state-automata-and-why-should-a-programmer-know-about-them) - -### What is the difference between finite state machine and finite automata? - -Both "Finite State Machine" FSM and "Finite Automata" (or Finite State Automata) FA means same, represents an abstract mathematical model of computation for the class of regular languages. - -The word "Finite" significance the presence of the finite amount of memory in the form of the finite number of states Q. - -Generally in formal-theory (or theory of computation), we prefer to use the word "Automata" – to emphasise that our machine is 'automatic' machine (self-moving: like our computer), "automatic" in the sense that once you have been defined transition rules, you do not need to apply any explicit intelligent to process strings (you just need to refer transition rules at each step). Remember our ultimate aim behind defining transition machines is to automate the computational task. - -By the way, automata or state-machines are a graphical representation to describe transition rules. - -You can also use "Transition Tables" or "Transition function" like `δ(q0, a) → q1`. Basically, all uses for the same purpose just to define "Mappings". - -[Source](https://stackoverflow.com/questions/22354706/can-anyone-please-explain-difference-between-finite-state-machine-and-finite-aut) - -### How does "δ:Q×Σ → Q" read in the definition of a DFA - -`×` means Cartesian product (that is a set), and `→` is a mapping. -`δ: Q×Σ → Q` says `δ` is a transition function that defined mapping from `Q×Σ` to `Q`. Where, Domain of `δ` is `Q×Σ` and Range is `Q`. - -Note: [Cartesian Product](http://en.wikipedia.org/wiki/Cartesian_product) itself a mathematical that all possible order pair (mapping) between two sets. - -You can also say: - -`δ` is a transition function that defined mapping between (or say associates) Cartesian product of set of states `Q` and language symbols `Σ` into set of state `Q`. This is abbreviated by `δ:Q×Σ → Q` - -Here, `Q` is finite set of states and `Σ` is a finite set of language symbols. - -Additionally in any automated you can represent transition function in tree ways. - -1. [Transition Table](http://en.wikipedia.org/wiki/State_transition_table#Common_forms) -2. [Transition graph](http://en.wikipedia.org/wiki/State_diagram) or say state diagram. -3. Transition function: a finite set of mapping rules. e.g. `{δ(q0, a) → q1, δ(q1, a) → q2}` - -In DFA. `δ:Q×Σ → Q` can also be written like `δ(Q,Σ) → Q` It's similar to function. In `δ` function two input arguments are state `Q` and a language symbol `Σ` and returned value is `Q`. - -**What is meaning of `δ(Q,Σ) → Q`** - -Suppose in your set of transition function δ you have an element `δ(q0, a) → q1` this means. If the present state is `q0` then by consuming a symbol you can shift to state `q1`. And the state-diagram for `δ(q0, a) → q1`: `(q0)---a---►(q1)` - -Some authors write `δ ⊆ Q×Σ → Q` in formal DFA definition that means `δ` is a Partial function (not defined on full Domain `Q×Σ`) - -[Source](https://stackoverflow.com/questions/14870130/how-does-%ce%b4q%c3%97%ce%a3%e2%86%92q-read-in-the-definition-of-a-dfa-deterministic-finite-automat?noredirect=1&lq=1) - -### State machine vs. workflow - -1/ The major difference between a workflow engine and a state machine lies in focus. In a workflow engine, a transition to the next step occurs when a previous action is completed, whilst a state machine needs an external event that will cause branching to the next activity. In other words, the state machine is event-driven and the workflow engine is not. [Source](https://workflowengine.io/blog/workflow-engine-vs-state-machine/) - -2/ A state machine (which is a map of states with transitions between them) would allow loops as opposed to a sequential workflow, which precedes down different branches until done. [Source](https://stackoverflow.com/questions/8840527/what-is-the-difference-between-state-machine-and-workflow?rq=3) - -### DFA vs. NFA - -#### DFA (Deterministic finite automaton) robot - -- This robot **only looks at one tile at a time** and knows **exactly what to do** next, no matter what. -- It has **one set of instructions** for each tile color. If it sees a red tile, it knows for sure what its next move is. -- It's **very strict** and follows only one route to figure out if the path is correct. - For example: - - If it sees a red tile, it moves forward. - - If it sees a blue tile, it turns around. It **never gets confused** and always knows the next step. - -#### NFA (Non-deterministic finite automaton) robot - -- This robot is a little different. When it sees a tile, it can **imagine multiple possibilities** and think about all of them at once. -- It might say, "Hmm, when I see a red tile, I could move forward, turn around, or even jump! Let me think about all these options at once." -- The robot **explores multiple paths** at the same time, as if it can split into multiple versions of itself. - For example: - - When it sees a red tile, it might think, "I can either move forward or jump over it." - - It checks **all options** at the same time and decides if the path is correct by looking at all the possibilities. - -#### Difference in capabilities - -- **DFA robot**: It's faster because it always knows exactly what to do. But it might need a lot of instructions because it can't explore different options. It has to account for every possible situation. -- **NFA robot**: It’s more flexible because it can explore lots of possibilities at the same time. But in the real world, it might take a little longer to check all those options. - -#### Key points - -- **DFA**: One option at a time, very efficient but can be strict. -- **NFA**: Many options at once, more flexible but can take more time to figure things out. diff --git a/topics/architecture/builder-design-pattern.md b/topics/architecture/builder-design-pattern.md deleted file mode 100644 index f796848ef..000000000 --- a/topics/architecture/builder-design-pattern.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: Introduce the builder pattern and its use cases -description: Builder, one of the creational patterns, allows user to construct complex object step by step while still maintaining flexibility. -date: 2024-06-10 -authors: - - tuanddd -tags: - - design-pattern - - gang-of-four -redirect: - - /IQSM9g ---- - -![](assets/builder-design-pattern.pdf) - -## Problem statement - -We want to create complex object without worrying too much about the hows, we could use Factory pattern to abstract away those details and just give us the output, but the drawback is you get a fixed same object everytime. There will be cases where you need to tweak some properties of that object to get the desired result, look no further than the Builder pattern. - -## Builder design pattern - -Create a Builder of the object you wanna build, e.g. QueryBuilder, ButtonBuilder, etc,... - -Add methods that act as steps to gradually build your object, this method must return an instance of the builder itself to allow method chainning (not required but a very distinct feature of this pattern). - -Have a method that finalizes and output the actual object, conventionally it's called "build()". - -Let's say we to build a Drink object, consider the following example of a DrinkBuilder class: - -```ts -class DrinkBuilder implements Builder { - private drink: Drink; - private name = ""; - private ingredients = []; - - constructor() {} - - name(n: string) { - this.name = n; - return this; - } - - addIngredient(name: string, ml: number) { - this.ingredients.push({ name, ml }); - return this; - } - - // where the actual building happens - mix() { - return new Drink(this.name, this.ingredients); - } -} -``` - -And then we can use it like this: - -```ts -new DrinkBuilder() - .name("JagerBomb") - .addIngredient("Jagermeister", 30) - .addIngredient("Redbull", 120) - .mix(); - -new DrinkBuilder() - .name("JagerGrenade") - .addIngredient("Jagermeister", 30) - .addIngredient("Tequila", 30) - .addIngredient("Vodka", 30) - .addIngredient("Redbull", 50) - .mix(); -``` - -Notice how by defining each step as method we (the consumer, one that uses the builder) can easily add/skip/modify certain properties of the object, this would all have been hidden away by the Factory class. And the method chaining helps with the readability too. - -## Real life examples - -jQuery was one of the early pioneers to adopt this pattern in their APIs, those little `$(something).on("click", func).toggle("class")...` is the Builder pattern in action. In fact, the javascript ecosytem is very fond of this pattern, some other libraries that make use of this are: - -- [wretch](https://github.com/elbywan/wretch) - a wrapper for native Fetch API with intuitive syntax -- [zod](https://github.com/colinhacks/zod) - a schema validation library -- [Spotify Nodejs wrapper](https://github.com/thelinmichael/spotify-web-api-node) - A wrapper for Spotify's web api -- and many others... - -## Pros & cons - -Pros: - -- Readablity: each step of the builder communicates clearly what it's doing and what parameters it requires. -- Flexibility: the ability to add X, modify Y, skip Z without modifying the internal logic. - -Cons: - -- For each object that you wanna build, you will need to create a separate Builder class for it, this can quickly become a problem as the codebase grows. - -## Reference - -- diff --git a/topics/architecture/c4-diagrams.md b/topics/architecture/c4-diagrams.md deleted file mode 100644 index 5d6699aae..000000000 --- a/topics/architecture/c4-diagrams.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: C4 diagrams -description: C4 diagrams use a four-level model to clearly visualize software system architecture, helping teams understand system context, containers, components, and code implementation. -date: 2022-04-17 -github_id: monotykamary -redirect: - - /hTiS1Q ---- - -## What are C4 diagrams? - -C4 diagrams, or the [C4 model](https://c4model.com/), is used to diagram domain and implementation abstractions of system software architectures. These diagrams represent system architectures in a semantically consistent format that can stand by itself with little to no prior context as they always incorporate a name, description, with relevant contexts such as type of technology or its deployment location. The semantics used for C4 diagrams are coincidentally very similar to [[State, Explain, Link]]. It is focused as an "abstraction-first" approach to diagramming, with the hierarchy of abstractions of the model going down 4 levels: - -1. **System context diagram** - shows the context of the entire system with related entities -2. **Container diagram** - a high level shape of the architecture and how it fits the IT environment -3. **Component diagram** - decompose containers into components to show implementation abstractions -4. **Code** - shows how the component is implemented as programmable code - -![](assets/c4-diagrams_c4-overview.webp) - -The C4 model isn't strictly static when it comes to these levels and also supports supplementary diagrams, such as **system landscape diagrams**, **dynamic diagrams**, **deployment diagrams**, etc. - -## The story of C4 diagrams - -The C4 model was created by Simon Brown, designed with the goal for it to be a developer friendly approach to diagramming. The high level representations of these diagrams incidentally also allowed it to assist in communication between software development teams and product teams. The abstractions make it suitable for software consulting and allows us to do architecture evaluations and risk identification. - -#### Reference - -- -- diff --git a/topics/architecture/command-pattern.md b/topics/architecture/command-pattern.md deleted file mode 100644 index 06080c966..000000000 --- a/topics/architecture/command-pattern.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: Command pattern -description: Command is a behavioral design pattern that encapsulates a request as an object. This allows you to parameterize methods with different requests, delay or queue a request’s execution, and support undoable operations. This pattern promotes the decoupling of the sender and receiver of a request, enhancing flexibility and maintainability. -date: 2024-06-27 -authors: - - vdhieu -tags: - - design-pattern - - gang-of-four -redirect: - - /UuOjTA ---- - -![Command pattern](assets/command-pattern_command-en-2x.webp) - -## What is the Command Design Pattern? - -Command is a behavioral design pattern that encapsulates a request as an object. This allows you to parameterize methods with different requests, delay or queue a request’s execution, and support undoable operations. This pattern promotes the decoupling of the sender and receiver of a request, enhancing flexibility and maintainability. - -## Key components of the Command pattern - -![Structure of the Command design pattern](assets/command-pattern_structure-2x.webp) - -### Command interface - -**Purpose** - -- Declares a method for executing a command. -- Ensures that all concrete commands implement this method, providing a consistent interface. - -**Responsibilities** - -- Defines an `execute()` method that `concrete commands` must implement. - -### Concrete command - -**Purpose** - -- Implements the Command interface. -- Defines the binding between a `Receiver` object and an action. -- Calls the appropriate operations on the `Receiver`. - -**Responsibilities** - -- Implements the `execute()` method by invoking the corresponding operation(s) on the `Receiver`. -- Holds a reference to the Receiver. - -### Receiver - -**Purpose** - -- Knows how to perform the operations needed to carry out the request. - -**Responsibilities** - -- Performs the actual work when its methods are called by the `Concrete command`. - -### Invoker - -**Purpose** - -- Asks the command to carry out the request. -- Can store and queue commands, and even support undo operations by storing executed commands. - -**Responsibilities** - -- Maintains a reference to a Command object (or a list of Command objects). -- Calls the `execute()` method on the Command object. - -## The real life example - -![Restaurent](assets/command-pattern_command-comic-1-2x.webp) - -> Imagine you own a small restaurant where you are both the chef and the person taking orders directly from your customers. As the chef, you prepare each meal yourself. -> -> As your restaurant gains popularity, you find it increasingly difficult to handle the growing number of orders. To manage this, you decide to hire a waiter. The waiter's responsibilities include taking orders from customers and writing them down on a piece of paper. -> -> The waiter then brings the written orders to the kitchen and sticks them on the wall. You, the chef, can pick up these order slips from the wall and prepare the meals accordingly. Once you finish cooking, you place the meal on a tray along with the corresponding order slip. -> The waiter retrieves the tray, double-checks the order, and serves the meal to the customer. - -In this scenario, the paper order serves as a `Command`. It remains in a queue until the chef is ready to prepare it. The order contains all the relevant information required to cook the meal, allowing the chef to start cooking immediately instead of clarifying the order details directly from the customer. - -## Pros and cons - -### Pros - -- Single Responsibility Principle: Decouples classes that invoke operations from classes that perform these operations. -- Open/Closed Principle: New commands can be introduced into the app without breaking existing client code. -- Undo/Redo: Supports implementing undo/redo functionality. -- Deferred Execution: Allows implementing deferred execution of operations. -- Complex Commands: Facilitates assembling a set of simple commands into a complex one. - -### Cons - -- Complexity: Introduces a whole new layer between senders and receivers, which can complicate the code. - -## Applicability - -- Transactional Systems: Financial/payment system, e-commerce platforms -- Task Scheduling: Job schedulers, cron jobs, and task queues, batch jobs, data processing pipelines -- User Interfaces: Optimistic UI - -## References - -- https://refactoring.guru/design-patterns/command diff --git a/topics/architecture/domain-model-in-domain-driven-design.md b/topics/architecture/domain-model-in-domain-driven-design.md deleted file mode 100644 index 0321e147c..000000000 --- a/topics/architecture/domain-model-in-domain-driven-design.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: Domain model in domain driven design -description: Learn what a Domain Model is in Domain Driven Design, including key components like Domain events, Commands, Aggregates, and Bounded Contexts to organize business knowledge effectively. -date: 2022-07-02 -github_id: R-Jim -redirect: - - /lDeHFw ---- - -## What is domain model - -In [[Overview of Domain Driven Design|Domain Driven Design]], Domain Model is the organized and structured **knowledge of the business problem**. Represents as a diagram, code examples, or written documentation, and **must** be accessible and understandable by **everyone** involved with the project. - -### Components of domain model - -The vocabulary and key concepts of the domain, and the relationships among all of the entities. These are mainly divided into **Domain events**, **Commands**, **Aggregates**, and **Bounded context**: - -### Domain events - -_A statement in past tense_ describes the **things that happened** in a business system that alternates the state of the entity. e.g: Order submitted, Shopping cart updated. - -### Commands - -_A verb in present tense_ describes the action that triggers the corresponding **domain event**. It is either user or system actions. - -- e.g: `Add product` (Command) -> `Shopping cart updated` (Domain event) - -### Aggregates - -Represented by a _minimal cluster_ of associated objects(domain events, commands, and actors) that we treat as a unit for data change. Each has a boundary and only exposes its root (**Aggregate Root**) which allows other objects to reference it. - -- e.g: A team wants to update its member role according to each project: `Project, Update member role` (Aggregate Root) -> `Project's member role updated` - -### Bounded context - -A high-level structure consists of categorizations of functionality, represents a circle or square, that groups related entities together. It can bound parts of an aggregate or multiple aggregates. - -- e.g: In an aggregate for the shopping process, we draw the bounded contexts for **Shopping cart**, and **Offers**: Shopping cart (`User` -> `Add product to cart` -> `Cart updated`) -> Offers (`Promotiational Offers Identified` -> `Offers added`) - -## References - -- https://herbertograca.com/category/development/book-notes/domain-driven-design-by-eric-evans/ -- Domain-driven design by Eric Evans -- https://creately.com/blog/diagrams/event-storming/ -- https://www.jamesmichaelhickey.com/domain-driven-design-aggregates/ -- https://serialized.io/java/working-with-aggregates/ diff --git a/topics/architecture/entities-in-domain-driven-design.md b/topics/architecture/entities-in-domain-driven-design.md deleted file mode 100644 index 078658cac..000000000 --- a/topics/architecture/entities-in-domain-driven-design.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Entities in domain driven design -description: Learn how Domain-Driven Design (DDD) uses entity identity and continuity to improve software by linking developers and domain experts for better system maintenance and scalability. -date: 2022-03-28 -github_id: R-Jim -redirect: - - /DB2iSA ---- - -## Recap on DDD - -A software design focusing on the **Domain**, one of the keys to a program success, by removing _communication lag_ between **Developers** and **Domain Experts**, _separate important Domain-specific components_ for maintenance and scalability. - -## A) Entities Defined by **Identity** - -_ID or Identity, identify 2 objects with the same attributes_ (2 persons with the same name, DoB but different ID card number). - -## B) Entities Defined by **Thread of continuity** - -_A set of fingerprints or traces of an object when going through multiple systems._ - -### Examples - -#### Real case world view - -- **Banking(A)**, a banknote(paper money) will have an ID that identifies it on the digital systems or a physical log book even if all of its attributes are the same as another banknote(value, printing date, printing batch, etc..). - -- **Online shopping service(A, B)**, an order has ID. - - After checkout, the system sends the order ID to the payment service. - - After payment completes, the system calls the Shipping service API to generate shipment with a tracking ID. - - On the shopping website, from the order detail page, the user can check payment, shipping status. In the payment, shipping system, order ID, and details can be viewed. When receiving the package, order ID, shipping fee, and total items, prices can be viewed. - -#### In-System implementation - -- **Generated ID(A)**. - -- **Combination of attributes(A)**, a batch of newspapers identify by name, city, and publish date. - -- **Logging Request time(B)**, when calling to a service, that service will log the request and identify it by request time in combination with an IP address (not recommended if the caller is a middleware service or gateway). - -## Entities's role in DDD - -_Establish continuity so that behavior can be clear and predictable_, We need to focus on how the system revolves around that entity and decide the identifier, not the reverse. - -- **A online shopping site for furniture**, the user only check brand name, model number, maybe design, and color for comparison between chairs or tables. Having a separate ID for each item might impact the decision on how to implement checkout(that item with ID is sold out so the user will need to find a similar item with a different ID). - -#### References - -- https://herbertograca.com/category/development/book-notes/domain-driven-design-by-eric-evans/ - -- Domain-driven design by Eric Evans diff --git a/topics/architecture/event-sourcing-overview.md b/topics/architecture/event-sourcing-overview.md deleted file mode 100644 index 79fe16980..000000000 --- a/topics/architecture/event-sourcing-overview.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -title: Event sourcing overview -description: Event sourcing records all changes as events, allowing e-commerce platforms to track detailed order history and generate flexible analytics for better business insights and future-proof data management. -date: 2022-12-05 -github_id: R-Jim -icy: 10 -redirect: - - /aBH0GQ ---- - -## What is event sourcing - -A database design revolves around **Events**. **Events** retain all business-critical information for auditing, market simulations, and future requirements that scale with the growth of the business. - -## Problem scenario: e-commerce platform - -We build an e-commerce platform. From all orders that successfully checked out, we generated a detailed report for sellers to improve their product's attraction to buyers. - -### Change to the requirement - -To be competitive with other platforms. We need to generate analytics for product retention when in the buyer's cart. So we need to have `order_log` to have data each time product's quantity changes in the cart. - -But we only have the analytics when the implementation launches in production, and all previous data before the implementation is lost. This cycle will repeat everytime a new requirement coming in. - -With the never-ending changes in business trends. We/Skateholders/Sellers/Buyers, can **never know which data is critical for the business survival in the future**. - -## Event sourcing - -### Event - -Event sourcing saves all information that happens in our system as **Events**. From the e-commerce platform scenario, an order's **event** is when something happens to the order (`order_created_event`, `product_added_event`, `product_removed_event`,...). - -An Event mainly consists of `Entity_ID`, `Type`, `Data`, `Metadata`, and `Version`: - -- `Entity_ID` is the identifier of the event's domain object. - Exp: `order: {id: 1}` -> `order_event: {entity_id: 1}` -- `Type` defines what happened to the domain object. - Exp: `order_created`, `product_added`, `product_removed`,... -- `Data` saves changes that happened to the domain object. - Exp: - -``` - product_added_event: { - data: [ - { product_id: book_1, quantity: 3 } - ] - } -``` - -- `Metadata` saves all user interactions and system metrics. - Exp: - -``` - product_added_event: { - metadata: { - user_ip: 192.168.1.1 - app: e-commerce-platform.v1-0-0 - device: macbook - response_time: 405ms - } - } -``` - -### Event Store - -A `database_table` stores events. **Only allowed Append** new event and **Read** from an event store. Event store naming will follow the domain object (`order_event_store`, `product_event_store`). - -### State - -When composing events, we get a State of a domain object. - -Exp: Get current products in order with `id == 1`. -We have list of `events` with `entity_id == 1`: - -``` - events: [ - { type: "product_added", data: [{ product_id: book, quantity: 1 }]}, - { type: "product_added", data: [{ product_id: book, quantity: 3 }, { product_id: pen, quantity: 2 }]}, - { type: "product_removed", data: [{ product_id: book, quantity: 2 }]} - ] -``` - -By composing all `events` we will have the `order` state with all current products: - -``` - order: { - id: 1, - products: [ - { product_id: book, quantity: 2 }, - { product_id: pen, quantity: 2 } - ] - } -``` - -With that same `events` we can compose a state for retention rate information: - -``` - order: { - id: 1, - products: [ - { product_id: book, retention_rate: 0.5 }, - { product_id: pen, retention_rate: 1 } - ] - } -``` - -These are **State Projection**. Through **Projector**, we present states that fit our business needs. - -And through our business process, we build states to validate if an event is valid to append to the event store with **Aggregation**. - -%%### Append event to Event store%% - -%%From FE, Client application's request, need to transform into an event%% - -%%### Read from Event store%% - -%%#### Projector - -Projector handle how we compose the event to get the usuable State of the domain oject - -#### Snapshot - -To reduce composing time when number of events start to grow. Snapshot is a State in a specifict time. To get the latest state, with a snapshot as base, with just need to compose rest of event from snapshot to now%% - -## References - -- [Beginner's guide to event sourcing | Event Store](https://www.eventstore.com/event-sourcing) -- [Keynote: event sourcing - Greg Young - DPC2016](https://www.youtube.com/watch?v=I3uH3iiiDqY) -- [Learning-topics: event sourcing](https://discord.com/channels/462663954813157376/1009812700022456400) -- [Diagram about the thought process from normal CRUD operation into event sourcing](https://miro.com/app/board/uXjVPZswY00=/?share_link_id=338629629501) diff --git a/topics/architecture/finite-state-automata.md b/topics/architecture/finite-state-automata.md deleted file mode 100644 index c9b5dc225..000000000 --- a/topics/architecture/finite-state-automata.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: Finite state automata -description: Learn what finite-state automata are, how they model systems using states and transitions, and see practical examples of state machines in programming with key concepts like states, events, and transitions. -date: 2022-06-28 -github_id: monotykamary -redirect: - - /i0OgDQ ---- - -## What are finite-state automata? - -A **finite state automaton (FSM)** (**FSA**, plural: automata), or better known as a state machine, is a mathematical model of with a constraint such that the abstract machine can be only and exactly be one of a finite number of states at any point in time. Finite state here is typically represented as a string or equivalent enumeration. - -## Mathematical model - -As per the general classification noted on the [Wikipedia page on finite-state machines](https://en.wikipedia.org/wiki/Finite-state_machine), deterministic finite-state machine has 5 main variables associated with its definition (a quintuple): $(\Sigma, S, s_0, \delta, F)$. - -- $\Sigma$ is the _input alphabet_ (a finite non-empty set of symbols) -> our events; -- $S$ is a finite non-empty set of states; -- $s_0$ is an _initial state_, an element of $S$; -- $\delta$ is the state-transition function: $\delta: S \times \Sigma \rightarrow S$; and -- $F$ is the set of _final states_, a (possibly empty) subset of $S$. - -Given an initial state $s_0$, to transition our state to the next state, our transition would simply be: - -$$ -\delta: s_0 \times \Sigma \rightarrow S -$$ - -If we were to reach a final state starting from a known set of states, our transition would look like: - -$$ -\delta: S \times \Sigma \rightarrow F -$$ - -### Simplified meaning - -For our purposes, we just need to understand that a `transition` function takes a `state` and an `event` to move on to a **new** `state`. What type of state and what subset it belongs to is up to the developer's discretion. - -```typescript -type State = string -type Event = string - -const transition = (state: State, event: Event): State => ... -``` - -## Coming from algebraic data types - -If you're coming from algebraic data types (ADTs), you may have noticed that state machines are nested ADTs that encode states and events as data types. In this case, we take advantage of nesting or combining pairs of sum/union types (example in Rescript): - -```typescript -type elapsed = float; - -type taskStatus = - | NotStarted - | Running - | Paused - | Done; - -type input = - | Start - | Pause - | Resume - | Finish; - -let transition = (state, input) => - switch (state, input) { - | (NotStarted, Start) => Running - | (Running, Pause) => Paused - | (Running, Finish) => Done - | (Paused, Resume) => Running - | (Paused, Finish) => Done - | (Running, Tick) => Running - | _ => state - }; -``` - -## States and relevance in the _domain_ - -In [[Domain Driven Design]] (DDD) and in [[Event storming]], there doesn't seem to be space for states in the domain. This is most likely because there is a separation of concern between the behavior of commands against what gets updated in the aggregate. DDD does allow classification of entities, such as classifying customer statuses, and these of course have business meaning. However, DDD doesn't specify any technical concerns such as the persistence of state or where a state transition should occur and what effects should happen. - -## References - -- https://en.wikipedia.org/wiki/Finite-state_machine -- https://wickstrom.tech/finite-state-machines/2017/11/10/finite-state-machines-part-1-modeling-with-haskell.html -- https://dev.to/margaretkrutikova/modelling-domain-with-state-machines-in-reasonml-n29 -- https://blog.honosoft.com/2019/10/31/partial-state-machine/ diff --git a/topics/architecture/finite-state-transducer.md b/topics/architecture/finite-state-transducer.md deleted file mode 100644 index 5c154650d..000000000 --- a/topics/architecture/finite-state-transducer.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: Finite state transducer -description: Learn what a finite-state transducer is, its mathematical model, and how it processes inputs to outputs with states, including examples in programming and applications in natural language processing. -date: 2022-06-28 -github_id: monotykamary -redirect: - - /mKfGnw ---- - -## What is finite-state transducer? - -It is essentially a [[Finite-state automata]] that has both inputs and outputs. In [Turing machines](https://en.wikipedia.org/wiki/Turing_machine), these inputs and outputs are referred to as 2 separate tapes: - -- **input tape**: a set of strings or related data. -- **output tape**: a set of relations generated from the input tape. - -This differs from a finite-state automaton which only has 1 (input) tape. - -## Mathematical model - -As per the general classification noted on [UC Davis outline on transducers](https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers) (formatted with similar variables to [[Finite-state automata]]s), a deterministic finite-state machine has 7 main variables associated with its definition (a septuple): ($\Sigma$, $S$, $\Gamma$, $\delta$, $\omega$, $s_0$, \_$F$). - -- $\Sigma$ is the _input alphabet_ (a finite non-empty set of symbols) -> our events; -- $S$ is a finite non-empty set of states; -- $\Gamma$ is the *output alphabet*; -- $\delta$ is the state-transition function: $\delta: S \times \Sigma \rightarrow S$ -- $\omega$ is the output-transition function: $\omega: S \times \Sigma \rightarrow \Gamma$ -- $s_0$ is an _initial state_, an element of $S$; -- $F$ is the set of *final states* and is a subset of $S$; and -- $\delta \subseteq S \times (\Sigma \cup \{\epsilon\}) \times (\Gamma \cup \{\epsilon\}) \times S$ (where ε is the [empty string](https://en.wikipedia.org/wiki/Empty_string "Empty string")) is the *transition relation*. - -Given any initial state in $s_0$, to transition our state to the next state with our output alphabet, our transition would be: - -$$ -\delta: s_0 \times \Sigma \rightarrow S -$$ - -$$ -\omega: s_0 \times \Sigma \rightarrow \Gamma -$$ - -$$ -\delta \subseteq s_0 \times (\Sigma \cup \{\epsilon\}) \times (\Gamma \cup \{\epsilon\}) \times S -$$ - -If we were to reach a final state starting from a known set of states, our transition would look pretty similar: - -$$ -\delta: S \times \Sigma \rightarrow F -$$ - -$$ -\omega: S \times \Sigma \rightarrow \Gamma -$$ - -$$ -\delta \subseteq S \times (\Sigma \cup \{\epsilon\}) \times (\Gamma \cup \{\epsilon\}) \times F -$$ - -Transducers that have a final state are used to recognize languages and have their use cases in [[Natural-Language Processing]]. - -### Simplified meaning - -There's actually quite a few ways to write up a transducer, at it is not always simply a beefed-up state machine. To oversimplify, we'll model it closer to a regular state machine: - -```typescript -type State = string -type Input = string -type Output = {...} // - -const transition = (state: State, input: Input): Output => ... -``` - -Outputs that are product types of itself and the next state of a transition is referred to as a [[Mealy machine]]. - -## Examples of basic transducers - -Although we've mentioned before that [[Reducers]] are single state machines, the canonical method of creating one mentioned in [Redux](https://redux.js.org/) and [React](https://reactjs.org/docs/hooks-reference.html#usereducer) are pretty much transducers as they can return state or objects (as **notions** of outputs). - -```typescript -// https://reactjs.org/docs/hooks-reference.html#usereducer -// our outputs here is the { count: number } object -const initialState = { count: 0 } - -function reducer(state, action) { - switch (action.type) { - case 'increment': - return { count: state.count + 1 } - case 'decrement': - return { count: state.count - 1 } - default: - throw new Error() - } -} - -function Counter() { - const [state, dispatch] = useReducer(reducer, initialState) - return ( - <> - Count: {state.count} - - - - ) -} -``` - -We can take our example from [[Finite-state automata]] and wrap it in a way that encapsulates both state, inputs and outputs (in Rescript): - -```typescript -// elapsed represents our arbitrary output -type elapsed = float; - -// elapsed here is used in a constructor as an arbitrary output -type taskStatus = - | NotStarted - | Running(elapsed) - | Paused(elapsed) - | Done(elapsed); - -// elapsed here is used in a constructor as an arbitrary input -type input = - | Start - | Pause - | Resume - | Finish - | Tick(elapsed); - -let transition = (state, input) => - switch (state, input) { - | (NotStarted, Start) => Running(0.0) - | (Running(elapsed), Pause) => Paused(elapsed) - | (Running(elapsed), Finish) => Done(elapsed) - | (Paused(elapsed), Resume) => Running(elapsed) - | (Paused(elapsed), Finish) => Done(elapsed) - | (Running(elapsed), Tick(tick)) => Running(elapsed +. tick) - | _ => state - }; -``` - -## Reference - -- https://en.wikipedia.org/wiki/Turing_machine -- https://t-pl.io/ddd-aggregates-processes-state-machines-and-transducers -- https://en.wikipedia.org/wiki/Finite-state_transducer -- https://reactjs.org/docs/hooks-reference.html#usereducer -- https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers -- https://dl.acm.org/doi/10.5555/972695.972698 -- https://web.stanford.edu/~laurik/publications/ciaa-2000/fst-in-nlp/fst-in-nlp.html diff --git a/topics/architecture/law-of-demeter.md b/topics/architecture/law-of-demeter.md deleted file mode 100644 index 824c87699..000000000 --- a/topics/architecture/law-of-demeter.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: Law of demeter -description: Learn how the Law of Demeter reduces object dependencies by promoting loose coupling and improving code flexibility, maintainability, and testability in object-oriented programming. -date: 2023-01-17 -github_id: namtrhg -icy: 10 -redirect: - - /jNbxnw ---- - -**The Law of Demeter (LoD)**, also known as the Philosophy of Least Knowledge, is a program design principle that states that an object should only communicate with objects that are close to it in the object graph. This concept contributes to reducing object dependency and making code more manageable and testable. - -## What is the Law of Demeter? - -This indicates that an object should not directly access the attributes or methods of another object to which it is not directly related. It should instead only invoke methods on objects to which it has a direct connection. - -For example, if object `A` wants to access a property of object `C`, it should not directly access the `C` property. Instead, it should request the property of `C` from object `B` to which it has a direct connection. - -This principle helps to promote loose coupling between objects, which makes the code more flexible and easier to change. It also makes it easier to test individual objects in isolation, as they are less dependent on other objects in the system. - -## Code example - -```js -class Info { - constructor(employee) { - this.employee = employee; - } - - // This violates the Law of Demeter - getEmployeeCompanyName() { - return this.employee.company.name; - } - - // This follows the Law of Demeter - getEmployeeComapnyName() { - return this.employee.getCompanyName(); - } -} - -class Employee { - constructor(name, company) { - this.name = name; - this.company = company; - } - - getCompanyName() { - return this.company.name; - } -} - -class Company { - constructor(name) { - this.name = name; - } -} - -const employee = new Employee("Nigel", new Company("Dwarves Foundation")); -const info = new Info(employee); -console.log(info.getEmployeeCompanyname()); // "Dwarves Foundation" -``` - -In this example, the Info class has a method `getEmployeeCompanyName()` which originally violates the Law of Demeter, as it directly accesses the `name` property of the `company` object. This creates a tight coupling between the Info and Company classes, making it more difficult to change or test one class without affecting the other. - -The Law of Demeter is followed by adding a new method `getCompanyName()` to the Employee class, which the Order class calls instead. The Info class no longer has to know anything about the Company class, it only needs to know that the employee has a `getCompanyName()` method. This makes the code more flexible and easier to change or test. - -## Pros and cons - -### Pros - -- Helps to minimize coupling between objects, making the code more modular and easier to maintain. -- Reduces the risk of unexpected side effects when making changes to the code. -- Makes it easier to understand the flow of data and control in the system. - -### Cons - -- This can lead to a large number of very small objects, which can make the code more difficult to understand. -- Can make it more difficult to enforce business rules that span multiple objects. -- This may lead to code duplication if the same data needs to be passed through several layers of objects. - -## What should you take into consideration? - -It's important to note that the Law of Demeter should be applied with discretion, as it is not always possible or desirable to completely eliminate all direct interactions between objects. The idea is to reduce the number of direct interactions as much as possible while still keeping the code readable and maintainable. - -## References - -- -- -- diff --git a/topics/architecture/mealy-machine.md b/topics/architecture/mealy-machine.md deleted file mode 100644 index 19af536ae..000000000 --- a/topics/architecture/mealy-machine.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: Mealy machine -description: Learn what a Mealy machine is, its mathematical model, and how it differs from Moore machines and finite-state transducers in this clear explanation of finite-state automata concepts. -date: 2022-06-28 -github_id: monotykamary -redirect: - - /iyzVHA ---- - -## What is a Mealy machine? - -A Mealy machine is a [[Finite-state automata]] where the output values are determined by its current state and current inputs. It is the closest definition to a deterministic [[Finite-state transducer]]. - -![](assets/mealy-machine_mealy_machine.webp) - -## Mathematical model - -As per the general classification noted on [UC Davis outline on transducers](https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers) (formatted with similar variables to [[Finite-state automata]]s), a deterministic Mealy machine has 6 main variables associated with its definition (a sextuple): ($\Sigma$, $S$, $\Gamma$, $\delta$, $\omega$, $s_0$). - -- $\Sigma$ is the _input alphabet_ (a finite non-empty set of symbols) -> our events; -- $S$ is a finite non-empty set of states; -- $\Gamma$ is the *output alphabet*; -- $\delta$ is the state-transition function: $\delta: S \times \Sigma \rightarrow S$ -- $\omega$ is the output-transition function: $\omega: S \times \Sigma \rightarrow \Gamma$ -- $s_0$ is an _initial state_, an element of $S$; and -- $\delta \subseteq S \times (\Sigma \cup \{\epsilon\}) \times (\Gamma \cup \{\epsilon\}) \times S$ (where ε is the [empty string](https://en.wikipedia.org/wiki/Empty_string "Empty string")) is the *transition relation*. - -Some formulations also allow transition and output functions to be combined as a single function: - -$$ -\delta: S \times \Sigma \rightarrow S \times \Gamma -$$ - -Given any initial state in $s_0$, to transition our state to the next state with our output alphabet, our transition would be: - -$$ -\delta: s_0 \times \Sigma \rightarrow S -$$ - -$$ -\omega: s_0 \times \Sigma \rightarrow \Gamma -$$ - -## Examples of basic Mealy machines - -Our example from [[Finite-state transducer]]s fits perfectly here as our transition and output function are coalesced as a single function. - -```typescript -// expiry represents our arbitrary output (in seconds) -type expiry = float; - -// expiry here is used in a constructor as an arbitrary output -type trafficLightStatus = - | Red(expiry) - | Amber(expiry) - | Green(expiry) - | FlashingRed(expiry) - -// elapsed here is used in a constructor as an arbitrary input -type input = - | ExpireTime - | Error - | Restart - -let transition = (state, input) => - switch (state, input) { - | (Red(expiry), ExpireTime) => Green(60.0) - | (Red(expiry), Error) => FlashingRed(30.0) - | (Green(expiry), ExpireTime) => Amber(60.0) - | (Green(expiry), Error) => FlashingRed(30.0) - | (Amber(expiry), ExpireTime) => Red(60.0) - | (Amber(expiry), Error) => FlashingRed(30.0) - | (FlashingRed(expiry), Restart) => Red(60.0) - | _ => state - }; -``` - -## Differences between - -### With formal [[Finite-state transducer]]s - -Mealy machines are a type of generator and are not used in processing language. As such, they do not have a concept of a final state. - -### With [[Moore machine]]s - -oth Mealy and Moore machines are generator-type state machines and can be used to parse [regular language](https://en.wikipedia.org/wiki/Regular_language). The outputs on a Mealy machine depend on **both the state and inputs**, whereas a Moore machine have their outputs **synchronously change with the state.** - -> Every Moore machine can be converted to a Mealy machine and every Mealy machine can be converted to a Moore machine. Moore machine and Mealy machine are equivalent. - -## Reference - -- https://en.wikipedia.org/wiki/Mealy_machine -- https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers -- https://unstop.com/blog/difference-between-mealy-and-moore-machine diff --git a/topics/architecture/moore-machine.md b/topics/architecture/moore-machine.md deleted file mode 100644 index 749398a13..000000000 --- a/topics/architecture/moore-machine.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: Moore machine -description: Learn about Moore machines, a type of finite-state machine where outputs depend only on current states, and explore their differences from Mealy machines and formal definitions. -date: 2022-06-28 -github_id: monotykamary -redirect: - - /TNr8jCE ---- - -## What is a Moore machine? - -A Moore machine is a [[Finite-state automata]] where the output values are determined by only its current state. Moore machines are a restricted type of a [[Finite-state transducer]]. - -![](assets/moore-machine_moore_machine.webp) - -## Mathematical model - -As per the general classification noted on [UC Davis outline on transducers](https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers) (formatted with similar variables to [[Finite-state automata]]s), a deterministic Moore machine has 6 main variables associated with its definition (a sextuple): ($\Sigma$, $S$, $\Gamma$, $\delta$, $\omega$, $s_0$). - -- $\Sigma$ is the _input alphabet_ (a finite non-empty set of symbols) -> our events; -- $S$ is a finite non-empty set of states; -- $\Gamma$ is the *output alphabet*; -- $\delta$ is the state-transition function: $\delta: S \times \Sigma \rightarrow S$ -- $\omega$ is the output-transition function: $\omega: S \rightarrow \Gamma$ -- $s_0$ is an _initial state_, an element of $S$; and -- $\delta \subseteq S \times (\Sigma \cup \{\epsilon\}) \times (\Gamma \cup \{\epsilon\}) \times S$ (where ε is the [empty string](https://en.wikipedia.org/wiki/Empty_string "Empty string")) is the *transition relation*. - -Given any initial state in $s_0$, to transition our state to the next state with our output alphabet, our transition would be: - -$$ -\delta: s_0 \times \Sigma \rightarrow S -$$ - -$$ -\omega: s_0 \rightarrow \Gamma -$$ - -## Examples of basic Moore machines - -Unlike a Mealy machine, we can't coalesce the transition and output functions together as a single transition function. The behavior of the output function is **synchronous** to the state change. As such, we end up with something like this (in Rescript): - -```typescript -type trafficLightStatus = - | Red - | Amber - | Green - | FlashingRed - -type input = - | ExpireTime - | Error - | Restart - -type outputFn = (state) => - switch (state) { - | Red => (Red, 60.0) - | Green => (Green, 60.0) - | Amber => (Amber, 60.0) - | FlashingRed => (FlashingRed, 30.0) - } - -let transitionFn = (state, input) => - switch (state, input) { - | (Red, ExpireTime) => Green(60.0) - | (Red, Error) => FlashingRed(30.0) - | (Green, ExpireTime) => Amber(60.0) - | (Green, Error) => FlashingRed(30.0) - | (Amber, ExpireTime) => Red(60.0) - | (Amber, Error) => FlashingRed(30.0) - | (FlashingRed, Restart) => Red(60.0) - | _ => state - }; - -let output = outputFn(transitionFn(Red, ExpireTime)) // (Green, 60.0) -``` - -## Differences between - -### With formal [[Finite-state transducer]]s - -Moore machines are a type of generator and are not used in processing natural language. As such, they do not have a concept of a final state. - -### With [[Mealy machine]]s - -Both Mealy and Moore machines are generator-type state machines and can be used to parse [regular language](https://en.wikipedia.org/wiki/Regular_language). The outputs on a Mealy machine depend on **both the state and inputs**, whereas a Moore machine have their outputs **synchronously change with the state.** - -> Every Moore machine can be converted to a Mealy machine and every Mealy machine can be converted to a Moore machine. Moore machine and Mealy machine are equivalent. - -## Reference - -- https://en.wikipedia.org/wiki/Mealy_machine -- https://www.cs.ucdavis.edu/~rogaway/classes/120/spring13/eric-transducers -- https://unstop.com/blog/difference-between-mealy-and-moore-machine diff --git a/topics/architecture/observer-pattern.md b/topics/architecture/observer-pattern.md deleted file mode 100644 index 4e2ebd607..000000000 --- a/topics/architecture/observer-pattern.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -title: Introduce the observer pattern and its use cases -description: Concept of the observer pattern with its pros and cons. -date: 2024-07-12 -authors: - - leduyhien152 -tags: - - observer-pattern - - design-pattern - - gang-of-four -redirect: - - /EcmfsQ ---- - -![](assets/observer-pattern.webp) - -## Problem - -Imagine that you have two types of objects: a Customer and a Store. The customer is very interested in a particular brand of product (say, it’s a new model of the iPhone) which should become available in the store very soon. - -The customer could visit the store every day and check product availability. But while the product is still en route, most of these trips would be pointless. - -![](assets/observer-pattern-problem.webp) - -On the other hand, the store could send tons of emails (which might be considered spam) to all customers each time a new product becomes available. This would save some customers from endless trips to the store. At the same time, it’d upset other customers who aren’t interested in new products. - -It looks like we’ve got a conflict. Either the customer wastes time checking product availability or the store wastes resources notifying the wrong customers. - -## Concept of the observer pattern - -### Definition - -The Observer pattern is a design pattern where an object, known as the subject, maintains a list of its dependents, called observers, and notifies them of state changes. - -### Real-world analogy - -If you subscribe to a newspaper or magazine, you no longer need to go to the store to check if the next issue is available. Instead, the publisher sends new issues directly to your mailbox right after publication or even in advance. - -The publisher maintains a list of subscribers and knows which magazines they’re interested in. Subscribers can leave the list at any time when they wish to stop the publisher sending new magazine issues to them. - -## Examples - -### DOM events and event listeners - -In the browser, we use the Observer pattern through DOM events. The DOM element is the subject, and event listeners are the observers. - -```js -button.addEventListener("click", () => { - console.log("Button clicked!"); -}); -``` - -### Custom event emitters - -We can create our custom event emitters to decouple code and enhance modularity. - -```js -const EventEmitter = require("events"); -const emitter = new EventEmitter(); - -emitter.on("event", () => { - console.log("An event occurred!"); -}); - -emitter.emit("event"); -``` - -In this example, `emitter` is the subject. When we emit the event, all registered observers get notified. - -### Redux: leveraging the observer pattern - -Redux is a state management library that uses the Observer pattern under the hood. -The Redux store is the subject, and components that subscribe to the store are the observers. - -```js -const { createStore } = require("redux"); - -const reducer = (state = {}, action) => { - switch (action.type) { - case "UPDATE": - return { ...state, data: action.payload }; - default: - return state; - } -}; - -const store = createStore(reducer); - -store.subscribe(() => { - console.log("State changed:", store.getState()); -}); - -store.dispatch({ type: "UPDATE", payload: "new data" }); -``` - -Here, the store dispatches actions, and subscribed components get notified to update their state. - -## Pros & cons - -Pros: - -- Loose Coupling: Reduces dependencies between subject and observers, making the system easier to maintain and extend. -- Automatic Updates: Observers are automatically notified and updated when the subject's state changes, keeping them in sync. -- Reusability and Flexibility: Observers and subjects can be reused independently. New observers can be added without modifying the subject. -- Event Handling: Provides a structured way to handle events and notify interested parties. -- Simplifies Communication: Abstracts the notification mechanism, simplifying communication between objects. - -Cons: - -- Potential for Memory Leaks: If observers are not properly unregistered, they can lead to memory leaks. -- Unexpected Updates: Observers may receive updates at unexpected times, leading to potential race conditions or inconsistent state. -- Performance Overhead: Notifying all observers can introduce performance overhead, especially if there are many observers or complex notifications. -- Complexity in Simple Scenarios: Can introduce unnecessary complexity for simple applications. -- Difficulty in Debugging: Tracking the flow of notifications and understanding the sequence of updates can be challenging. - -## Reference - -- https://refactoring.guru/design-patterns/observer diff --git a/topics/architecture/overview-of-domain-driven-design.md b/topics/architecture/overview-of-domain-driven-design.md deleted file mode 100644 index ac8254d4f..000000000 --- a/topics/architecture/overview-of-domain-driven-design.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: Overview of domain driven design -description: Domain Driven Design (DDD) improves software by creating clear domain models and shared language between developers and business experts, helping solve complex business problems effectively. -date: 2022-06-21 -github_id: R-Jim -redirect: - - /rT8p8Q ---- - -### What is Domain Driven Design? - -A software design approach focusing on the **Domain**, one of the keys to an application success, by creating a rich and meaningful **Domain model** using rules and conventions like **Ubitious language**, **Event storming**, etc. - -By enforcing the conversation around the domain. It removes _communication lag_ between **Developers** and **Domain/Business experts**. - -### Removal of vocabulary ambiguity - -Domain expert speaks in business term while developer speaks in technical. DDD introduces **Ubiquitous language**, a common rigorous medium, build between both parties to define statements, software solutions, etc, without any ambiguity - hence the term _rigorous_. - -### Better understanding of the business domain - -Through the drawn out **Domain model, Domain events** important aspects of the business are clarified, speculated for potential features, issues, and critical business flow can be prioritized for enhancement and scalability. - -### Technology independent - -The core of DDD is about the design decisions and transitions that were made in modeling the domain. So, without being too involved in the technical aspects, the development team has more options to select or adopt new technology. - -### Human aspects - -With the output diagrams and conversation, stakeholders/PMs have a better statistic to measure the success of the project. - -From the resources management side, It reduces the time for newcomers to grasp the overall system by discarding most translation documentation for business/technical terms and promoting discussions with others. - -And for developers, It is always important to understand the problems that we are using technology to solve. - -### When to use DDD - -DDD is designed to tackle complex business domains so it might not be the best for applications with minor domain complexity but high technical complexity. Required discipline and dedicated development team and domain experts. - -Here are some example domains that used DDD: - -- https://github.com/ibm-cloud-architecture/vaccine-solution-main (distribution) -- https://github.com/ddd-by-examples/library (booking). - -### References - -- https://herbertograca.com/category/development/book-notes/domain-driven-design-by-eric-evans/ -- Domain-driven design by Eric Evans diff --git a/topics/architecture/overview-on-broker-pattern-in-distributed-system.md b/topics/architecture/overview-on-broker-pattern-in-distributed-system.md deleted file mode 100644 index 644554b14..000000000 --- a/topics/architecture/overview-on-broker-pattern-in-distributed-system.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Overview on broker pattern in distributed system -description: Learn how the broker pattern structures distributed systems by decoupling clients and servers, enabling scalable communication through proxies and brokers for dynamic, independent agent interaction. -date: 2019-08-24 -tags: - - distributed -redirect: - - /8gOaVo ---- - -## Definition - -The broker pattern is an architectural pattern that can be used to structure distributed software systems with decoupled components that interact by remote procedure calls. A broker component is responsible for coordinating communication, such as forwarding requests, as well as for transmitting results and exceptions. - -## Component - -1. Clients: implements user functionality, send requests to server through a client-side proxy -1. Server: Implements services, registers itself with the local broker, send responses and exceptions back the client through a server-side proxy -1. Broker(a messenger that is responsible for the transmission of requests from clients to servers): registers server, offers interface(APIs, ...), transfer messages, error recovery, interoperates with other brokers through bridges, locates servers -1. Client-side proxy(a layer between clients and the broker): encapsulates system-specific functionality, mediates between the client and the broker -1. Server-side proxy: calls services within the server, encapsulates system-specific functionality, mediates between server and the broker -1. Bridge( responsible for communication among brokers): encapsulates network-specific functionality, mediates between the local broker and the bridge of a remote broker - -![](assets/overview-on-broker-pattern-in-distributed-system_e4d47aa7182bbec713b6dc4f858fb1dd_md5.webp) - -## The role of broker patter - -- Our system need to be dynamically removing or adding new agent(server, client) -- The agents in our system need the ability to be independent of each other( scalable, partition functionality into independent agents) - -## The ability of broker pattern - -In the decoupled behavior, Broker acts like an interface, when a new server was adding on the system, it just like add a new object, and the only thing we need to do is to register the new server with Broker. - -Based on the idea of the independent agent(server, client), Broker pattern give us free control to individual agents(it doesn't care about what we do with agents). Thus, the agent freely from scaling itself has its functionality diff --git a/topics/architecture/prototype-design-pattern.md b/topics/architecture/prototype-design-pattern.md deleted file mode 100644 index 2e6854f07..000000000 --- a/topics/architecture/prototype-design-pattern.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: Going through use cases of the prototype design pattern and it place among the creational patterns -description: "Prototype, one of the creational patterns, minimize efforts when recreating new from the exist object by cloning the 'prototype' of it." -date: 2024-06-10 -authors: - - R-Jim -tags: - - gang-of-four - - graphical-notation - - software-design -redirect: - - /V6ZMUA ---- - -![](assets/prototype-design-pattern.pdf) - -## Problem statement - -We want a copy of an object, but the config only initialized at runtime, its fields and methods were private, or its properties were manipulated through multiple processes, so it is difficult to recreate the object. For example: - -- DB Access Object with credentials only provide in runtime. -- A ledger object has transaction histories, which are protected through private fields, and exposes a Total() function to get the total balance. To replicate the whole ledger, we must recreate all its transaction histories. - -## Prototype design pattern - -Request a Clone of the object(Prototype) without the need to look up its class and implementation. - -The target object must turn into a 'Prototype' by having a Clone() function to mirror the object with sufficient properties for the user of the cloned object. For example: - -``` -//The following example is in Go. We are building an RPG with a Hero object, and a new Skill called 'Mimic' needs to create a copy of the Hero with the same level. -type Hero struct { // level, experience, and killLogIDs are private to avoid editing the Hero object - level int64 - experience int64 - killLogIDs []int64 -} - -type (h *Hero) KillConfirm(targetID int64) { - h.killLogIDs = append(killLogIDs, targetID) - h.experience += 10 - - if h.experience >= 100 { - h.level += 1 - h.experience -= 100 - } -} - -// Creates a clone of the Hero without repeating the whole kill log -type (h *Hero) Clone() Hero { - return Hero { - level: h.level, - experience: h.experience, - killLogIDs: h.killLogIDs, - } -} -``` - -## Case by case - -When developing a system, the Prototype design pattern works in tandem with other creation patterns: - -- Factories, abstract factories, and builders help create the original object. Prototype provides a clone of the object without the need to call the above patterns -- Prototype behaves opposite to the singleton pattern. Singleton pattern focuses on having a single instance for operational flow. Prototype can Clone the object in the middle of its processing flow for other uses without changing the original object. - -The most common use cases for Prototype design patterns are: - -- Credential/Access objects for Database or external services -- Ledger/Bank statement objects, we clone the object to the statistics and simulations. - -## Reference - -- https://refactoring.guru/design-patterns/prototype diff --git a/topics/architecture/sdk-event-sourcing.md b/topics/architecture/sdk-event-sourcing.md deleted file mode 100644 index d7e16f9b8..000000000 --- a/topics/architecture/sdk-event-sourcing.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Sdk event sourcing -description: Learn how Software Development Kits (SDKs) provide tools and APIs for app development, and discover how Event sourcing manages data communication between systems efficiently. -date: 2020-05-21 -tags: - - event -redirect: - - /Q9Py0A ---- - -## Software Development Kit (SDK) - -SDK stands for Software Development Kit: the tools and software to develop application through a specific platform. SDK provides libraries, document, templates, code sample, debugging, supportive documentation or additional document for developers to integrate into their software/ application. SDK can also be a combination of many APIs under the form of libraries to interact with the operating system. - -Let's take a cellphone for an example. Every cellphone has an operating system with programmed API, and many underlying connected libraries. A kit of APIs that can be used for different hardwares with the same operating system. - -## Function of SDK - -- Interact with hardware -- Interact with files -- Manage cookies - -## Event sourcing - -Event sourcing is a term for an architecture that is designed for interaction/ communication between app A and app B. Event sourcing acts as a message-oriented middleware where data is stored and sorted on prioritized order, and ready to be called out for communication purpose between parties. - -Consider Event sourcing as a mailbox to store the data for communication. - -- event: stands for data -- sourcing: where the data comes from - -### Example: TIKI - -Tiki is a marketplace for good tradings. Let's say TIKI has 3 types of parties - -**Marketplace** - -- Time of order -- Time of invoice - -**Warehouse** - -- Stock status -- Time of import/ export - -**Support** - -- Delivery status -- Goods returning - -The data between parties has a connection with each other, and it needs interaction to smooth the process of goods handling and delivery. Different types of data (events) from different parties (source) will need a system to filter and organize it with prioritized order to make sure the communication goes smoothly, that's what Event sourcing is all about. \_\_ - -## React hook - -Hooks are functions that let you “hook into” React state and lifecycle features from function. diff --git a/topics/architecture/service-based-architecture.md b/topics/architecture/service-based-architecture.md deleted file mode 100644 index fe252f67f..000000000 --- a/topics/architecture/service-based-architecture.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Service based architecture -description: Service-based architecture offers a flexible, middle-ground approach between monoliths and microservices, ideal for managing complex business needs without the overhead of full microservice complexity. -date: 2022-06-21 -github_id: monotykamary -redirect: - - /E67zHg ---- - -## What is service-based architecture? - -Service-based architecture is a kind of hybrid or middle-ground architecture between microservices and a monolith and is noted as a pragmatic architecture style due to its flexibility. Like a microservice architecture, it is essentially a distributed architecture, but it doesn't come with the cost of or complexity of other distributed architectures. - -![](assets/service-based-architecture_pasted-image-20220922153254.webp) - -### What are the differences between microservices, service-oriented, and service-based architectures? - -> Microservices Architecture and Service-Oriented Architecture (SOA) are considered service-based architectures - -Mark Richards, one of the authors who helped coin the term in the book _Fundamentals of Software Architecture: An Engineering Approach_, notes that service-based architectures lie as a **superset** of microservice and service-oriented architectures. The most notable patterns shared between the architectures, service contracts and a reliance on the [[BASE model]] with regard to database transactions. - -## Why use a service-based architecture? - -It is quite arguably a one-size fits a lot of stuff architecture. It is very suitable for projects that contain business requirements a bit too complex to manage (or otherwise accrue technical debt) with a monolith. Likewise, it also doesn't require the level of loose coupling you would typically see in a strict microservice environment. - -#### References - -- Fundamentals of Software Architecture: An Engineering Approach—by Mark Richards, Neal Ford -- https://microservices.io/ -- https://en.wikipedia.org/wiki/Microservices -- https://nofluffjuststuff.com/magazine/2015/10/the_challenges_of_service_based_architecture diff --git a/topics/architecture/singleton-design-pattern.md b/topics/architecture/singleton-design-pattern.md deleted file mode 100644 index ac27ae5a6..000000000 --- a/topics/architecture/singleton-design-pattern.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -title: A tour of Singleton design pattern with Golang -description: Singleton real-world problem, concept, solution, use cases, implementations, pros & cons, references -date: 2024-06-10 -authors: - - anhnh12 -tags: - - design-pattern - - gang-of-four - - golang - - software-design -redirect: - - /us0xGQ ---- - -![](assets/singleton-design-pattern.pdf) - -## Problem - -Just imagine we need to build a simple web page to show live subscribers of a Youtube channel. Main object here is "subscriber" so we need a tool (counter) to update number of subscribers in real-time. - -There are 2 actors: subscribers (Youtube) and visitors (our web page) - -When a user subscribes to a channel, we update the counter by +1. And we do the same for other incoming subscribers, but there are 2 things we have to keep in mind: - -- Handle concurrent subscription requests from users properly. Or else there will be a chance that we misscount a portion of subscribers -- Use the same counter to update/get live subscriptions for all actors (subscribers & visitors) - -## Singleton overview - -**Singleton** is one of **Creational design patterns**. It has some characteristics: - -- One class/type can only create/have one single instance -- Provide a global access to that single instance -- We have many Singleton implementations to be used in single/multi-thread environment based on specific use case - -## What does Singleton resolve? - -Back to our above analogy with subscription counter, we can use Singleton to instatiate the counter and provide its global access. - -- All subscribers and visitors will access the same counter so data will be consistent -- Since this is a multi-thread use case (e.g. multiple subscribers can subscribe simultaneously), we need to make sure the counter instantiation thread-safe - -## Applicability - -Use Singleton when you need to manage one & only instance of a resource (e.g. configuration, logger, etc.) - -## Approaches - -There are 2 ways to implement Singleton pattern: - - - - - - - - - - - - - - - - - - -
EagerLazy
Initialize the instance as soon as your app is startedInitialize the instance as the first time it is requested (e.g. the counter is only instantiated when the first subscription is made)
Should only be applied to light-weight resource since it will take a lot of compute power at the beginning and reduce the app performanceShould not be applied to heavy-size resource since it will unnecessary take a lot of compute power at the beginning and reduce the app performance
Simple to implement, don't have to worry about race conditionMore complex implementation, need to make sure the instantiation is thread-safe to avoid creating redundant instances
- -## Pseudocode (golang) - -First, define `counter` struct - -```go -type counter struct { - views int -} -``` - -
- -**Eager initialization** - -```go -var instance *counter = &counter{} - -func getCounter() *counter { - return instance -} -``` - -
- -**Lazy initialization**: use _double-checked locking mechanism_ to avoid unnecessary lock when the instance has already been initialized - -```go -var instance *counter - -func getCounter() *counter { - // instance has not been initialized - if instance == nil { - // lock to avoid multiple instances are created simultaneously by multi threads - lock.Lock() - defer lock.Unlock() - - // need to recheck because first check can be passed by multiple gorountines - if instance == nil { - instance = &counter{} - } - } - - // when instance has already been initialized -> return - return instance -} -``` - -## Benefits & drawbacks - -**Benefits**
-You can assure that the class/type has only one instance if the implementation is done properly. - -**Drawbacks**
- -- Tight coupled codebase
-- Hard to debug
-- Write unit tests will be tricky
-- Limited use cases
- -## References - -- https://refactoring.guru/design-patterns/singleton diff --git a/topics/architecture/solid-principles.md b/topics/architecture/solid-principles.md deleted file mode 100644 index de9f9279e..000000000 --- a/topics/architecture/solid-principles.md +++ /dev/null @@ -1,347 +0,0 @@ ---- -title: Solid principles -description: Learn the SOLID principles of object-oriented design to create clean, maintainable, and flexible code with key concepts like Single Responsibility, Open-Closed, Liskov Substitution, Interface Segregation, and Dependency Inversion. -date: 2023-04-20 -github_id: nguyend-nam -icy: 10 -redirect: - - /Pqjh0A ---- - -## What & Why? - -The SOLID Principles are five principles of **object-oriented** class design. They are a set of rules and best practices to follow while designing a class structure. Even though the acronym "SOLID" was introduced by Michael Feathers, The concept of those 5 principles were first introduced by the famous Computer Scientist [Robert C. Martin](https://en.wikipedia.org/wiki/Robert_C._Martin) (a.k.a Uncle Bob) in 2000. - -Uncle Bob is also known for his masterpieces of _Clean Code_ and _Clean Architecture_, etc. Therefore it is not a surprise that all these concepts of clean coding, object-oriented architecture, and design patterns are somehow connected and complementary to each other. - -> See Uncle Bob's [publications](https://en.wikipedia.org/wiki/Robert_C._Martin#Publications). - -Each principle of SOLID will be explained with example below: - -- [[#S - single responsibility principle]] -- [[#O - open-closed principle]] -- [[#L - liskov substitution principle]] -- [[#I - interface segregation principle]] -- [[#D - dependency inversion principle]] - -## S - single responsibility principle - -Let's begin with the single responsibility principle. As we all figured out from the name, this principle states that **a class should only have one responsibility and therefore it should only have one reason to change.** - -For example, let's look at a class to represent a simple book: - -```cpp -class Book { - private: - string name; - string author; - unsigned int publicYear; -} -``` - -In this code, we store the name, author and public year associated with an instance of a `Book`. Let's now add a couple of methods to query the text: - -```cpp -class Book { - private: - string name; - string author; - unsigned int publicYear; - - public: - // direct initialization constructor - Book(string n, string a, unsigned int p) : name(n), author(a), publicYear(p) {} - - string getName() const { - return name; - } - - string getAuthor() const { - return author; - } - - unsigned int getPublicYear() const { - return publicYear; - } -}; - -class Printer { - public: - void printCitation(const Book &book) { - cout << book.getName() << " (" << book.getAuthor() << ", " - << book.getPublicYear() << ")" << endl; - } -}; -``` - -In the example above, we have 2 classes. The `Book` is responsible for representing a book object and having methods for users to access its attributes. The class `Printer` will also have a **single responsibility**, which is printing. One example can be taken into account is printing the citation: - -```cpp -int main() { - Book book("Where the Wild Things Are", "Sendak", 1963); - Printer printer; - - printer.printCitation(book); - // Where the Wild Things Are (Sendak, 1963) - return 0; -} -``` - -Some developers might define the method `printCitation` inside the `Book` class, but this code violates the single responsibility principle. When following the practice as the code snippets above, not only the code is much cleaner, but we will also separate those classes by concerns ([SoC - Separation of Concerns](https://en.wikipedia.org/wiki/Separation_of_concerns#:~:text=In%20computer%20science%2C%20separation%20of,code%20of%20a%20computer%20program.)). Later on, if there are classes representing the arts or videos that we may need to print their information, we can leverage the `Printer` class implemented before by adding methods such as `artInfoPrinter`. - - - -## O - open-closed principle - -The open-closed principle states that classes, modules, and functions should be **open for extension, but closed for modification**. It means you should be able to extend the functionality of them by adding more code without modifying the existing code. - -The code below violates this principle: - -```cpp -class Animal { - private: - string name; - string type; - unsigned int legs; - - public: - Animal(string n, string t, unsigned int l) : name(n), type(t), legs(l) {} - - void getSpeed() { - if(type == "cheetah"){ - cout << "130mph" << endl; - } else if (type == "lion"){ - cout << "80mph" << endl; - } else if (type == "elephant"){ - cout << "40mph" << endl; - } else { - cout << "Unsupported animal type" << endl; - } - } -}; -``` - -The problem in the above code is that if we want to add new animal types, we have to modify the existing code by adding cases into the `switch` statement of the method `getSpeed`. To fix this, we can refactor as below: - -```cpp -// Base class -class Animal { - private: - string name; - string type; - unsigned int legs; - - public: - Animal(string n, string t, unsigned int l) : name(n), type(t), legs(l) {} - - // unimplemented pure virtual method - virtual void getSpeed() = 0; -}; - -// Derived classes -class Cheetah : public Animal { - public: - Cheetah(string n, string t, unsigned int l) : Animal(n, t, l) {} - - void getSpeed() override { - cout << "130mph" << endl; - } -}; - -class Lion : public Animal { - public: - Lion(string n, string t, unsigned int l) : Animal(n, t, l) {} - - void getSpeed() override { - cout << "80mph" << endl; - } -}; - -int main() { - Lion lion("lion", "cat", 4); - lion.getSpeed(); - // 80mph -} -``` - -By creating a brand new class for the new behavior, we would know that the stuff we already built isn't affected and we can totally focus on designing the class to suit the new requirement. - - - -## L - liskov substitution principle - -The Liskov Substitution principle is one of the most important principles to adhere to in object-oriented programming (OOP). It states that child classes or subclasses must be substitutable for their parent classes or super classes. Narrowing it down, we have **if class A is a subclass of class B, we should be able to replace B with A without disrupting the behavior of our program.** - -```cpp -// Base class -class MeansOfTransport { - public: - virtual void turnOnEngine() { - cout << "Turn on the engine" << endl; - } -}; - -// Derived classes -class Motorbike : public MeansOfTransport {}; - -// Printer class -class MeansOfTransportPrinter { - public: - void printEngineAction(MeansOfTransport &t){ - t.turnOnEngine(); - } -}; -``` - -As usual we define the base class with several derived classes and an additional printer class. - -```cpp -int main() { - Motorbike motorbike; - MeansOfTransportPrinter printer; - - printer.printEngineAction(motorbike); - // Turn on the engine - return 0; -} -``` - -The method `printEngineAction` of the `MeansOfTransportPrinter` accepts the param of `MeansOfTransport` type. As we defined the derived class `Motorbike` from `MeansOfTransport`, we can also pass that subclass as a param to the `printEngineAction` method. But the Liskov Substitution principle may be violated in the following situation: - -```cpp -// Derived classes -class ElectricCar : public MeansOfTransport { - public: - void turnOnEngine() override { - cout << "What engine??? You mean motor?" << endl; - } -}; - -int main() { - ElectricCar electricCar; - MeansOfTransportPrinter printer; - - printer.printEngineAction(electricCar); - // What engine??? You mean motor? - return 0; -} -``` - -The idea behind the Liskov Substitution principle is that a derived class should be able to replace its base class in any code that uses the base class, without causing unexpected behavior or violating any assumptions made about the base class. In the example above, the `ElectricCar` is defined to be a derived class from `MeansOfTransport`, but unlike the engine in the base class, it uses an electrical motor. It could be either mistaken when defining attributes or methods in the base class or when we leverage it and make the derived class, but after all, the use above is an example that violates the Liskov Substitution principle. - - - -## I - interface segregation principle - -According to this principle, **a client should never be forced to implement an interface that it doesn’t use**, or a client shouldn’t be forced to depend on methods it does not use. More specifically, the principle suggests that software developers should break down large interfaces into smaller, more specific ones that are independent of other interfaces that are not relevant to them. - -Think about this as the same thing we do while working with [Micro-Frontend](https://dwarvesf.hashnode.dev/micro-frontend-what-why) architecture, we usually break down the codebase into views and furthermore, into components that hold specific responsibilities. - -```cpp -// Base classes -class PersonGeneralInfo { - private: - string name; - string gender; - unsigned int age; - - public: - // Constructor and methods to get private attributes -}; - -class PersonWorkingInfo { - private: - string company; - unsigned int salary; - - public: - // Constructor and methods to get private attributes -}; - -// Derived classes -class Baby : public PersonGeneralInfo {}; -// Baby just need to inherit from PersonGeneralInfo, -// stuff relating to salary or company makes no sense - -class Adult : public PersonGeneralInfo, public PersonWorkingInfo {}; -``` - -## D - dependency inversion principle - -This principle is about **decoupling modules, making them as separate from one another as possible**. The principle states that high-level modules should not depend on low-level modules. Instead, they should both depend on abstractions. - -Imagine we are having an application that uses the logger to log messages. Sometimes we just need to log to the console, but in some cases we want to export them to a text file, forming a short report. - -```cpp -// Base class -class ILogger { - public: - virtual void log(string message) = 0; -}; - -// Derived classes -class ConsoleLogger : public ILogger { - public: - void log(string message) override { - cout << message << endl; - } -}; - -class FileLogger : public ILogger { - public: - void log(string message) override { - ofstream file; - file.open("log.txt"); - file << message << endl; - file.close(); - } -}; -``` - -Next let's define a class for our application: - -```cpp -class App { - private: - ILogger& logger; - - public: - App(ILogger& logger) : logger(logger) {} - - void run() { - logger.log("App started"); - } -}; -``` - -Now let's add these lines of code to our main function: - -```cpp -int main() { - ConsoleLogger consoleLogger; - FileLogger fileLogger; - - App appWithConsoleLogger(consoleLogger); - appWithConsoleLogger.run(); - - App appWithFileLogger(fileLogger); - appWithFileLogger.run(); - return 0; -} -``` - -We can see the message has been logged into a "log.txt" file and another one in the console. Now let's dive into the code. The **abstract** class `ILogger` is responsible for the base of all the logger approaches, in this example we have `ConsoleLogger` and `FileLogger`. Our application will be able to takes in any logger and based on the one we provide, the message will then be export to the file or logged into the console. - - - -## Benefits - -We have taken a deep dive into the SOLID principles of object-oriented design. How do these principles help us to build better software? They encourage us to create more **maintainable**, **scalable**, and **flexible** software. As our applications grow in size, we can reduce their complexity and lower the effort we need to put to scaling and maintaining. - -Applying the Single Responsibility or Liskov Substitution principle helps us to keep track of the functionality of each module, boosts the process of testing and threfore makes the applications less likely to have unexpected behaviors. As for the principles like Open-Closed, Interface Segregation or Dependency Inversion, they make sure we create reusable components, reduce the coupling between different modules and increase flexibility of our system. - -## Reference - -- https://www.freecodecamp.org/news/solid-design-principles-in-software-development/ -- https://www.baeldung.com/solid-principles -- https://dev.to/galwaycoder/the-solid-principles-in-software-design-explained-53n diff --git a/topics/architecture/state-pattern.md b/topics/architecture/state-pattern.md deleted file mode 100644 index 4199a0a1e..000000000 --- a/topics/architecture/state-pattern.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: State pattern -description: The State Design Pattern is a behavioral design pattern that allows an object to change its behavior when its internal state changes. This pattern is particularly useful for scenarios where an object can exist in multiple states and its behavior varies based on these states. -date: 2024-06-27 -authors: - - vdhieu -tags: - - design-pattern - - gang-of-four -redirect: - - /3Mke5Q ---- - -![State pattern](assets/state-pattern_state-en-2x.webp) - -> The State Design Pattern is a behavioral design pattern that allows an object to change its behavior when its internal state changes. This pattern is particularly useful for scenarios where an object can exist in multiple states and its behavior varies based on these states. - -## Example scenario - -Consider a Document class with three states: Draft, Moderation, and Published. The publish method behaves differently in each state: - -- Draft: Moves the document to Moderation. -- Moderation: Publishes the document if the current user is an administrator. -- Published: Does nothing. - -![Document state change](assets/state-pattern_problem2-en-2x.webp) - -### Common implementation issues - -```py -class Document is - field state: string - // ... - method publish() is - switch (state) - "draft": - state = "moderation" - break - "moderation": - if (currentUser.role == "admin") - state = "published" - break - "published": - // Do nothing. - break - // ... -``` - -- Monstrous Conditionals: As the number of states and transitions increases, the conditional logic becomes complex and difficult to manage. -- Maintenance Difficulty: Changes to the state transitions require updating conditionals in multiple places, increasing the risk of errors -- Scalability Issues: Predicting all possible states and transitions at the design stage is challenging, and the state machine can become bloated over time as new states and behaviors are added. - -### Solution using State pattern - -The State pattern addresses these issues by encapsulating state-specific behavior into separate state classes. This way, the context class delegates the behavior to the state objects, making the code cleaner and easier to manage. - -![Solution](assets/state-pattern_solution-en-2x.webp) - -## Concept of State pattern - -![Finite-State Machine](assets/state-pattern_problem1-2x.webp) - -In the State Design Pattern, the state of an object is represented by a set of state-specific classes. The object, known as the context, delegates state-specific behavior to the current state object. As the state of the context changes, it transitions between different state objects, each of which implements a particular set of behaviors. - -### Structure - -![Structure](assets/state-pattern_structure-en-2x.webp) - -- **Context**: Maintains an instance of a `ConcreteState` subclass that defines the current state. -- **State**: Defines an interface for encapsulating the behavior associated with a particular state of the Context. -- **ConcreteState**: Implements the behavior associated with a state of the Context. - -### Advantages - -- **Simplifies state transitions**: Encapsulates state-specific behavior and state transitions, making it easier to add new states without modifying existing ones. -- **Enhances readability**: Improves code readability and maintainability by organizing state-specific behavior into separate classes. -- **Promotes open/closed principle**: Facilitates adherence to the Open/Closed Principle, allowing the system to be extended with new states without altering existing code. - -### Use cases - -The State Design Pattern is particularly beneficial in scenarios such as: - -- **Finite state machines**: Implementing state machines where an object can be in one of a limited number of states. -- **User interfaces**: Managing different UI states like enabled, disabled, focused, etc. -- **Game development**: Handling various game states such as running, paused, stopped, etc. diff --git a/topics/architecture/strategy-design-pattern.md b/topics/architecture/strategy-design-pattern.md deleted file mode 100644 index fa65fe802..000000000 --- a/topics/architecture/strategy-design-pattern.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: Strategy design pattern, the concept, use cases and difference with the state design pattern -description: Strategy design pattern, a behavioral design pattern that denote the functionality of a family of interchangeable classes to a interface, the context, with the helper objects, selects the appropriate implementation of the interface. -date: 2024-07-11 -authors: - - R-Jim -tags: - - design-pattern - - gang-of-four -redirect: - - /9c8wow ---- - -![](assets/strategy-design-pattern.pdf) - -## Problem statement - -The separation of a renowned cookbook, by cuisine culture, into distinct cookbooks(strategies on how to cook dishes): The owner restaurant of the cookbook keeps a single cookbook to prevent the leaking of trade secrets. To maintain standards, the head cooks must use the cookbook to follow the complicated recipes. When the restaurant grows, conflicts happen between head cooks taking turns to use the single cookbook. - -=> The cookbook divides into multiple cookbooks based on their type(appetizer, main dish, side dish, ...) and cuisine culture(asian, french, italy, ...). The head cook selects the cook books(strategies) by their station and their dish's cuisine. - -## `Strategy` design pattern - -Strategy design pattern is about grouping classes with similar functionality, an interface is created to represent their functions, and each class has implementation following the interface. - -To choose the correct strategy, the context uses the provided information and selects the appropriate implementation/strategy of the interface. - -Example: - -```go -// Given an interface to group classes with similar functionality -type Calculation interface { - func Calculate(x, y int) int // calculate two numbers: x and y, return int result -} - -// Implementations -type CalculationAdd Calculation - -// Calculate returns the addition result of x and y -func (c CalculationAdd) Calculate(x, y int) int { - return x + y -} - -type CalculationMinus Calculation - -// Calculate returns the subtraction result of x and y -func (c CalculationMinus) Calculate(x, y int) int { - return x - y -} - -// The program calculator, from the input operator, calculates two numbers and prints the result -func main() { - var x, y int - x, y = env.GetInputNumbers() - var operation string - operation = env.GetInputOperation() - - var calculation Calculation - // Select calculation - switch operation { - case "ADD": - calculation = CalculationAdd - case "Minus": - calculation = CalculationMinus - } - - log.Printf("Result of operation %d %s %d is: %d, x, operation, y, calculation.Calculate(x, y)) -} -``` - -## Use cases - -Strategy design patterns commonly used in systems with diverse business flows, a few examples: - -- Route planning system that changes based on the provided vehicle type. -- Transaction system that deals with different types of charging, subscription, and discount. - -## Vs `State` design pattern - -State can be considered as an extension of Strategy. Both patterns are based on composition. -They change the behavior of the context by delegating some work to helper objects. - -| Strategy | State | -| --------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| An interface is created by the similarity of function, the helper objects select the appropriate strategy | Each class represents a 'State'. The helper objects transition the context to its state and then perform the State behavior | -| Object strategy completely independent and unaware of each other | Doesn't restrict dependencies, may aware and initiate transitions from one state to another | - -## Reference - -- https://refactoring.guru/design-patterns/strategy diff --git a/topics/architecture/uml-state-machine-diagram.md b/topics/architecture/uml-state-machine-diagram.md deleted file mode 100644 index 7af160650..000000000 --- a/topics/architecture/uml-state-machine-diagram.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: Uml state machine diagram -description: Learn how UML State Machine Diagrams visualize object states, transitions, events, and actions to model dynamic system behavior in software development, embedded systems, and more. -date: 2023-08-16 -github_id: trankhacvy -icy: 10 -redirect: - - /dDz2CQ ---- - -## UML state machine diagram - -Unified Modeling Language (UML) is a powerful tool used in software engineering to visualize, design, and communicate the structure and behavior of systems. Among its various diagram types, the UML State Machine Diagram stands out as a fundamental representation of the dynamic behavior of an object or system. In this article, we will delve into the intricacies of UML State Machine Diagrams, exploring their components, symbols, applications, and benefits. - -### Introduction to UML state machine diagrams - -At their core, UML State Machine Diagrams capture the state transitions and behaviors of objects or entities within a system. These diagrams provide a clear representation of how an object's state changes in response to events and conditions, ultimately leading to a better understanding of the system's behavior. UML State Machine Diagrams are particularly useful in modeling complex behaviors, especially those that involve multiple states and intricate transitions. - -### Components of UML state machine diagrams - -1. **States:** A state represents a condition or mode that an object can be in. For example, in a traffic light system, states could include "Red," "Yellow," and "Green." Each state is depicted as a rounded rectangle with the state's name inside. - -2. **Transitions:** Transitions depict the change from one state to another due to events or conditions. These are represented by arrows connecting the states and are labeled with the triggering event or condition that leads to the transition. Transitions can also have associated actions or effects that occur when the transition takes place. - -3. **Events:** Events are occurrences that trigger transitions between states. These can be external events like user inputs or internal events like timers reaching a certain value. - -4. **Actions:** Actions are behaviors or activities that are executed when a transition occurs. They can be associated with transitions to specify what happens during the transition, aiding in understanding the system's behavior. - -5. **Guards:** Guards are conditions that must be satisfied for a transition to occur. They are often depicted as expressions associated with transitions, ensuring that the appropriate conditions are met before a state transition happens. - -6. **Initial and final states:** An initial state indicates the starting point of the state machine, while a final state marks the end point. An object may enter the final state after reaching a specific state or completing its task. - -### Symbols in UML state machine diagrams - -1. **State:** Rounded rectangle with the state's name inside. -2. **Transition:** Arrow connecting states, labeled with the triggering event/condition. -3. **Event:** Named occurrence that triggers transitions. -4. **Action:** Activity that occurs during a transition. -5. **Guard:** Condition that must be fulfilled for a transition to happen. -6. **Initial state:** Filled circle indicating the initial state. -7. **Final state:** Bullseye-like symbol representing the final state. - -![](assets/uml-state-machine-diagram.webp) - -### Applications of UML state machine diagrams - -UML State Machine Diagrams find applications across various domains: - -1. **Software development:** In software engineering, these diagrams model the behavior of objects or components within a system. They assist developers in understanding and implementing complex state transitions and behaviors. - -2. **Embedded systems:** For systems like IoT devices or hardware controllers, state machine diagrams help designers visualize how the system responds to different inputs and events. - -3. **Game development:** Games often involve complex character behaviors and interactions. State machine diagrams aid in designing the characters' states, animations, and responses to player actions. - -4. **Business processes:** State machine diagrams can represent the lifecycle of a business process or workflow, helping to identify bottlenecks and areas of improvement. - -5. **Communication protocols:** When designing communication protocols or network systems, state machine diagrams are invaluable for illustrating how devices or systems react to various messages and conditions. - -### Benefits of using UML state machine diagrams - -1. **Clarity and understanding:** These diagrams provide a visual representation of dynamic behavior, making it easier for stakeholders to understand how a system operates. - -2. **Complexity management:** UML State Machine Diagrams help manage complex behaviors by breaking them down into states, transitions, and actions, simplifying the design process. - -3. **Requirements validation:** By mapping states and transitions to requirements, developers can ensure that the system fulfills the intended behaviors and functionalities. - -4. **Effective communication:** UML State Machine Diagrams serve as a common language between developers, designers, testers, and clients, facilitating effective communication and collaboration. - -5. **Documentation:** These diagrams serve as comprehensive documentation for the system's behavior, aiding in future maintenance and updates. - -### Conclusion - -UML State Machine Diagrams are a powerful tool for modeling and understanding the dynamic behavior of systems, objects, and entities. By visually representing states, transitions, events, and actions, these diagrams offer a clear picture of how a system behaves under different conditions. Their applications in software development, embedded systems, game design, and more make them a crucial asset for designers, developers, and stakeholders alike. With their ability to simplify complexity and improve communication, UML State Machine Diagrams continue to play a pivotal role in modern software engineering and system design. - -### References - -- https://en.wikipedia.org/wiki/UML_state_machine -- https://www.visual-paradigm.com/guide/uml-unified-modeling-language/what-is-state-machine-diagram/ -- https://sparxsystems.com/resources/tutorials/uml2/state-diagram.html -- https://www.lucidchart.com/pages/uml-state-machine-diagram diff --git a/topics/architecture/visitor-design-pattern.md b/topics/architecture/visitor-design-pattern.md deleted file mode 100644 index ffd47683b..000000000 --- a/topics/architecture/visitor-design-pattern.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -title: Visitor design pattern, the concept, problem solution and use cases -description: Visitor is a behavioral design pattern that lets you separate algorithms from the objects on which they operate. -date: 2024-07-12 -authors: - - taipham1803 -tags: - - design-pattern - - gang-of-four -redirect: - - /7XpmUw ---- - -![](assets/visitor-design-pattern.pdf) - -![illustration](assets/visitor-design-pattern-1.webp) - -## What is the Visitor Design Pattern? - -**Visitor** is a behavioral design pattern that lets you separate algorithms from the objects on which they operate. - -Visitor design pattern is one of the [**behavioral design patterns**](https://www.geeksforgeeks.org/software-design-patterns/). It is used when we have to perform an operation on a group of similar kind of Objects. With the help of visitor pattern, we can move the operational logic from the objects to another class. The visitor pattern consists of two parts: - -- a method called **Visit()** which is implemented by the visitor and is called for every element in the data structure -- visitable classes providing **Accept()** methods that accept a visitor - -## Problem statement - -Imagine you're developing a simple text editing application. In this application, you have different types of document elements such as paragraphs, tables, and images. Each of these elements can perform certain operations, such as rendering to the screen, exporting to different formats (like HTML or plain text), and spell-checking. - -As the application grows, more operations need to be supported for these elements. Without a proper design, adding new operations could lead to a bloated and hard-to-maintain codebase, especially if it involves modifying each element class every time a new operation is introduced. - -## Solution with visitor pattern - -The Visitor Pattern allows us to define a new operation without changing the classes of the elements on which it operates. Instead of adding the new operation to each element, we create a visitor class that implements the operation. Each element class then has an `accept` method that takes a visitor and calls the visitor’s method corresponding to that element. - -## Structure - -- The Visitor interface declares a set of visiting methods that can take concrete elements of an object structure as arguments. These methods may have the same names if the program is written in a language that supports overloading, but the type of their parameters must be different. -- Each Concrete Visitor implements several versions of the same behaviors, tailored for different concrete element classes. - -![structure](assets/visitor-design-pattern-2.webp) - -## Code example for problem - -### Illustration - -```tsx - +-----------------+ - | DocumentVisitor | - +-----------------+ - | +visitParagraph | - | +visitTable | - | +visitImage | - +-----------------+ - /|\ - | - +----------------+----------------+ - | | | -+-----------------+ +-----------------+ +-------------------+ -| RenderVisitor | | ExportVisitor | | SpellCheckVisitor | -+-----------------+ +-----------------+ +-------------------+ -| +visitParagraph | | +visitParagraph | | +visitParagraph | -| +visitTable | | +visitTable | | +visitTable | -| +visitImage | | +visitImage | | +visitImage | -+-----------------+ +-----------------+ +-------------------+ - - | - | - V - +---------------------------+ - | DocumentElement | - +---------------------------+ - | +accept(visitor: Visitor) | - +---------------------------+ - /|\ - | - +------------+------------+ - | | -+-----------------+ +-----------------+ +-----------------+ -| Paragraph | | Table | | Image | -+-----------------+ +-----------------+ +-----------------+ -| +accept(visitor)| | +accept(visitor)| | +accept(visitor)| -+-----------------+ +-----------------+ +-----------------+ -``` - -### Element classes - -```tsx -interface DocumentElement { - accept(visitor: DocumentVisitor): void; -} - -class Paragraph implements DocumentElement { - accept(visitor: DocumentVisitor): void { - visitor.visitParagraph(this); - } -} - -class Table implements DocumentElement { - accept(visitor: DocumentVisitor): void { - visitor.visitTable(this); - } -} - -class Image implements DocumentElement { - accept(visitor: DocumentVisitor): void { - visitor.visitImage(this); - } -} -``` - -### Visitor interface and Concrete visitors: - -```tsx -interface DocumentVisitor { - visitParagraph(paragraph: Paragraph): void; - visitTable(table: Table): void; - visitImage(image: Image): void; -} - -class RenderVisitor implements DocumentVisitor { - visitParagraph(paragraph: Paragraph): void { - console.log("Rendering a paragraph."); - } - - visitTable(table: Table): void { - console.log("Rendering a table."); - } - - visitImage(image: Image): void { - console.log("Rendering an image."); - } -} - -class ExportVisitor implements DocumentVisitor { - visitParagraph(paragraph: Paragraph): void { - console.log("Exporting a paragraph to HTML."); - } - - visitTable(table: Table): void { - console.log("Exporting a table to HTML."); - } - - visitImage(image: Image): void { - console.log("Exporting an image to HTML."); - } -} - -class SpellCheckVisitor implements DocumentVisitor { - visitParagraph(paragraph: Paragraph): void { - console.log("Spell checking a paragraph."); - } - - visitTable(table: Table): void { - console.log("Spell checking a table."); - } - - visitImage(image: Image): void { - console.log("Spell checking an image."); - } -} -``` - -### Usage - -```tsx -function App() { - const documentElements: DocumentElement[] = [ - new Paragraph(), - new Table(), - new Image(), - ]; - - const renderVisitor = new RenderVisitor(); - const exportVisitor = new ExportVisitor(); - const spellCheckVisitor = new SpellCheckVisitor(); - - for (const element of documentElements) { - element.accept(renderVisitor); - element.accept(exportVisitor); - element.accept(spellCheckVisitor); - } -} -``` - -### Explanation - -1. **Element classes:** We have `Paragraph`, `Table`, and `Image` classes, each implementing the `accept` method which accepts a visitor. -2. **Visitor interface:** `DocumentVisitor` is an interface with methods to visit each type of element. -3. **Concrete visitors:** `RenderVisitor`, `ExportVisitor`, and `SpellCheckVisitor` are concrete implementations of the visitor interface. Each visitor class defines the operation for each type of element. -4. **Usage:** We create instances of elements and visitors. Each element accepts each visitor, which performs the appropriate operation. - -By using the Visitor Pattern, we can easily add new operations without modifying the element classes, adhering to the Open/closed principle and making the code more maintainable and scalable. - -## Applicability - -- **Use the Visitor pattern to perform operations on all elements of a complex object structure (e.g., an object tree).** - This pattern allows you to execute an operation across a set of objects of different classes by having a visitor object implement multiple variants of the same operation, tailored to each target class. -- **Use the Visitor to simplify the business logic by separating auxiliary behaviors.** - It helps keep the primary classes of your app focused on their main responsibilities by moving other behaviors into separate visitor classes. -- **Use the Visitor when a behavior is relevant only to certain classes in a class hierarchy.** - Extract this behavior into a separate visitor class, implementing only the visiting methods for the relevant classes, leaving the rest empty. - -## Pros and cons - -### Advantages - -- **Open/closed principle**: Introduce new behavior for objects of different classes without modifying those classes. -- **Single responsibility principle**: Consolidate multiple versions of the same behavior into a single class. -- A visitor object can gather useful information while working with various objects, which is beneficial for traversing complex structures like an object tree and applying the visitor to each object. - -### Disadvantages - -- All visitors need updating whenever a class is added to or removed from the element hierarchy. -- Visitors may lack access to private fields and methods of the elements they work with. - -## Use cases - -Some use cases for the Visitor Pattern: - -### Use case 1: compiler design - -**Context:** In a compiler, the abstract syntax tree (AST) represents the structure of the source code. The compiler needs to perform various operations on the AST, such as type checking, code generation, and optimization. - -**Solution:** The Visitor Pattern can be used to define these operations without changing the classes representing the AST nodes. - -**Example:** - -- **Elements:** `Expression`, `Statement`, `Variable`, `Function` -- **Visitors:** `TypeChecker`, `CodeGenerator`, `Optimizer` - -### Use case 2: document processing - -**Context:** In a text processing application, different document elements (e.g., paragraphs, images, tables) need to support various operations like rendering, exporting, and spell-checking. - -**Solution:** The Visitor Pattern allows new operations to be added without modifying the element classes. - -**Example:** - -- **Elements:** `Paragraph`, `Table`, `Image` -- **Visitors:** `RenderVisitor`, `ExportVisitor`, `SpellCheckVisitor` - -### Use case 3: graphics rendering - -**Context:** In a graphics rendering system, different shapes (e.g., circles, squares, triangles) need to support operations like drawing, resizing, and calculating the area. - -**Solution:** The Visitor Pattern can be used to add these operations without altering the shape classes. - -**Example:** - -- **Elements:** `Circle`, `Square`, `Triangle` -- **Visitors:** `DrawVisitor`, `ResizeVisitor`, `AreaCalculatorVisitor` - -### Use case 4: file system operations - -**Context:** In a file system management tool, different file system components (e.g., files, directories) need to support operations like searching, compression, and encryption. - -**Solution:** The Visitor Pattern allows these operations to be added without changing the component classes. - -**Example:** - -- **Elements:** `File`, `Directory` -- **Visitors:** `SearchVisitor`, `CompressionVisitor`, `EncryptionVisitor` - -### Use case 5: game development - -**Context:** In a game, different game entities (e.g., players, enemies, obstacles) need to support various operations like rendering, updating state, and collision detection. - -**Solution:** The Visitor Pattern can be used to define these operations without modifying the entity classes. - -**Example:** - -- **Elements:** `Player`, `Enemy`, `Obstacle` -- **Visitors:** `RenderVisitor`, `UpdateVisitor`, `CollisionDetectionVisitor` - -### Use case 6: e-commerce system - -**Context:** In an e-commerce application, different product types (e.g., electronics, clothing, groceries) need to support operations like applying discounts, calculating shipping costs, and generating invoices. - -**Solution:** The Visitor Pattern allows new operations to be added without changing the product classes. - -**Example:** - -- **Elements:** `Electronics`, `Clothing`, `Groceries` -- **Visitors:** `DiscountVisitor`, `ShippingCostVisitor`, `InvoiceGeneratorVisitor` - -### Use case 7: network protocols - -**Context:** In a network protocol implementation, different types of packets (e.g., data packet, acknowledgment packet, control packet) need to support operations like serialization, deserialization, and logging. - -**Solution:** The Visitor Pattern can be used to add these operations without modifying the packet classes. - -**Example:** - -- **Elements:** `DataPacket`, `AckPacket`, `ControlPacket` -- **Visitors:** `SerializeVisitor`, `DeserializeVisitor`, `LoggingVisitor` - -### Use case 8: UI component management - -**Context:** In a GUI application, different UI components (e.g., buttons, text fields, checkboxes) need to support operations like rendering, event handling, and validation. - -**Solution:** The Visitor Pattern allows these operations to be added without changing the component classes. - -**Example:** - -- **Elements:** `Button`, `TextField`, `Checkbox` -- **Visitors:** `RenderVisitor`, `EventHandlingVisitor`, `ValidationVisitor` diff --git a/topics/blockchain/assert-require-revert.md b/topics/blockchain/assert-require-revert.md deleted file mode 100644 index 193a2ba27..000000000 --- a/topics/blockchain/assert-require-revert.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: Assert require revert -description: Learn how to enforce smart contract invariants in Solidity using assert(), require(), and revert() to ensure error checking and secure contract behavior. -date: null -redirect: - - /EomPiQ ---- - -#### Enforce invariants with `assert()` - -An assert guard triggers when an assertion fails - such as an invariant property changing. For -example, the token to ether issuance ratio, in a token issuance contract, may be fixed. You can -verify that this is the case at all times with an `assert()`. Assert guards should often be -combined with other techniques, such as pausing the contract and allowing upgrades. (Otherwise, you -may end up stuck, with an assertion that is always failing.) - -Example: - -```sol -contract Token { - mapping(address => uint) public balanceOf; - uint public totalSupply; - - function deposit() public payable { - balanceOf[msg.sender] += msg.value; - totalSupply += msg.value; - assert(address(this).balance >= totalSupply); - } -} -``` - -Note that the assertion is _not_ a strict equality of the balance because the contract can be forcibly sent ether without going -through the `deposit()` function! - -#### Use `assert()`, `require()`, `revert()` properly - -!!! Info -The convenience functions **assert** and **require** can be used to check for conditions and throw an exception if the condition is not met. - -The **assert** function should only be used to test for internal errors, and to check invariants. - -The **require** function should be used to ensure valid conditions, such as inputs, or contract state variables are met, or to validate return values from calls to external contracts. \* - -Following this paradigm allows formal analysis tools to verify that the invalid opcode can never be -reached: meaning no invariants in the code are violated and that the code is formally verified. - -```sol -pragma solidity ^0.5.0; - -contract Sharer { - function sendHalf(address payable addr) public payable returns (uint balance) { - require(msg.value % 2 == 0, "Even value required."); //Require() can have an optional message string - uint balanceBeforeTransfer = address(this).balance; - (bool success, ) = addr.call.value(msg.value / 2)(""); - require(success); - // Since we reverted if the transfer failed, there should be - // no way for us to still have half of the money. - assert(address(this).balance == balanceBeforeTransfer - msg.value / 2); // used for internal error checking - return address(this).balance; - } -} -``` - -See [SWC-110](https://swcregistry.io/docs/SWC-110) & [SWC-123](https://swcregistry.io/docs/SWC-123) diff --git a/topics/blockchain/assets/blockchain-bridge-external-links.webp b/topics/blockchain/assets/blockchain-bridge-external-links.webp deleted file mode 100644 index 2541ad02f..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge-external-links.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge.webp b/topics/blockchain/assets/blockchain-bridge.webp deleted file mode 100644 index 8e13d48a3..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_blockruption-blockchain-300h.webp b/topics/blockchain/assets/blockchain-bridge_blockruption-blockchain-300h.webp deleted file mode 100644 index dd349be90..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_blockruption-blockchain-300h.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_bridge-use-cases.webp b/topics/blockchain/assets/blockchain-bridge_bridge-use-cases.webp deleted file mode 100644 index 90202607b..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_bridge-use-cases.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_classify-bridge.webp b/topics/blockchain/assets/blockchain-bridge_classify-bridge.webp deleted file mode 100644 index 91b1a4b4d..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_classify-bridge.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_interoperability-dilemma.webp b/topics/blockchain/assets/blockchain-bridge_interoperability-dilemma.webp deleted file mode 100644 index f0bd1bb1f..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_interoperability-dilemma.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_lightweight-client-&-relays.webp b/topics/blockchain/assets/blockchain-bridge_lightweight-client-&-relays.webp deleted file mode 100644 index 7b09683b3..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_lightweight-client-&-relays.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_lock,-mint-and-burn.webp b/topics/blockchain/assets/blockchain-bridge_lock,-mint-and-burn.webp deleted file mode 100644 index 65f10041b..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_lock,-mint-and-burn.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-bridge_offchain-actors.webp b/topics/blockchain/assets/blockchain-bridge_offchain-actors.webp deleted file mode 100644 index 64017034e..000000000 Binary files a/topics/blockchain/assets/blockchain-bridge_offchain-actors.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_19b7383d98cf76a0b76e3c19de34d88e_md5.webp b/topics/blockchain/assets/blockchain-for-designers_19b7383d98cf76a0b76e3c19de34d88e_md5.webp deleted file mode 100644 index 6c9cbeb4a..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_19b7383d98cf76a0b76e3c19de34d88e_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_60e714810139595dfd5ee4591022b6cd_md5.webp b/topics/blockchain/assets/blockchain-for-designers_60e714810139595dfd5ee4591022b6cd_md5.webp deleted file mode 100644 index 21704f106..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_60e714810139595dfd5ee4591022b6cd_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_83a093e62fa61b069e3ea4b2f22b407e_md5.webp b/topics/blockchain/assets/blockchain-for-designers_83a093e62fa61b069e3ea4b2f22b407e_md5.webp deleted file mode 100644 index 33862357b..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_83a093e62fa61b069e3ea4b2f22b407e_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_a533a30cdbb6139ad9f5f98a57b9b558_md5.webp b/topics/blockchain/assets/blockchain-for-designers_a533a30cdbb6139ad9f5f98a57b9b558_md5.webp deleted file mode 100644 index 56ecb1225..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_a533a30cdbb6139ad9f5f98a57b9b558_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_bbdedf1451e87a0024ee6e71a138c663_md5.webp b/topics/blockchain/assets/blockchain-for-designers_bbdedf1451e87a0024ee6e71a138c663_md5.webp deleted file mode 100644 index 93a095185..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_bbdedf1451e87a0024ee6e71a138c663_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_d86795011c5ab5910ec0b5f6b55c74fe_md5.webp b/topics/blockchain/assets/blockchain-for-designers_d86795011c5ab5910ec0b5f6b55c74fe_md5.webp deleted file mode 100644 index 7b5770580..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_d86795011c5ab5910ec0b5f6b55c74fe_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_e1bfa96832cd4042e417869a7c426a77_md5.webp b/topics/blockchain/assets/blockchain-for-designers_e1bfa96832cd4042e417869a7c426a77_md5.webp deleted file mode 100644 index 71dc0455b..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_e1bfa96832cd4042e417869a7c426a77_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_f7c25751bae3d7b1b87f6c7b94d141d2_md5.webp b/topics/blockchain/assets/blockchain-for-designers_f7c25751bae3d7b1b87f6c7b94d141d2_md5.webp deleted file mode 100644 index 9e5a48c72..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_f7c25751bae3d7b1b87f6c7b94d141d2_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-for-designers_fc723d7bffd27c3988c8b5ea62194db0_md5.webp b/topics/blockchain/assets/blockchain-for-designers_fc723d7bffd27c3988c8b5ea62194db0_md5.webp deleted file mode 100644 index e6aff8b2e..000000000 Binary files a/topics/blockchain/assets/blockchain-for-designers_fc723d7bffd27c3988c8b5ea62194db0_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/blockchain-oracle_ins_and_outs_of_the_blockchain_ecosystem.webp b/topics/blockchain/assets/blockchain-oracle_ins_and_outs_of_the_blockchain_ecosystem.webp deleted file mode 100644 index cc06d4a7e..000000000 Binary files a/topics/blockchain/assets/blockchain-oracle_ins_and_outs_of_the_blockchain_ecosystem.webp and /dev/null differ diff --git a/topics/blockchain/assets/blocks_eqhpghw.webp b/topics/blockchain/assets/blocks_eqhpghw.webp deleted file mode 100644 index acc2679ea..000000000 Binary files a/topics/blockchain/assets/blocks_eqhpghw.webp and /dev/null differ diff --git a/topics/blockchain/assets/blocks_lzrylgx.webp b/topics/blockchain/assets/blocks_lzrylgx.webp deleted file mode 100644 index f4950142e..000000000 Binary files a/topics/blockchain/assets/blocks_lzrylgx.webp and /dev/null differ diff --git a/topics/blockchain/assets/blocks_qi5jtrd.png.webp b/topics/blockchain/assets/blocks_qi5jtrd.png.webp deleted file mode 100644 index c79276c23..000000000 Binary files a/topics/blockchain/assets/blocks_qi5jtrd.png.webp and /dev/null differ diff --git a/topics/blockchain/assets/blocks_svihd1p.webp b/topics/blockchain/assets/blocks_svihd1p.webp deleted file mode 100644 index 46b1d72c3..000000000 Binary files a/topics/blockchain/assets/blocks_svihd1p.webp and /dev/null differ diff --git a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_flow.webp b/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_flow.webp deleted file mode 100644 index 53c8d4126..000000000 Binary files a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_flow.webp and /dev/null differ diff --git a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_intro.webp b/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_intro.webp deleted file mode 100644 index 498f259dd..000000000 Binary files a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_intro.webp and /dev/null differ diff --git a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_result.gif b/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_result.gif deleted file mode 100644 index 83a2954a4..000000000 Binary files a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_result.gif and /dev/null differ diff --git a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_system.webp b/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_system.webp deleted file mode 100644 index 950f87e40..000000000 Binary files a/topics/blockchain/assets/build_custom_ai_agent_with_elizaos_system.webp and /dev/null differ diff --git a/topics/blockchain/assets/cross-chain-transfers-implementing-a-token-swap-from-base-chain-to-bitcoin-1.png b/topics/blockchain/assets/cross-chain-transfers-implementing-a-token-swap-from-base-chain-to-bitcoin-1.png deleted file mode 100644 index 3743741a1..000000000 Binary files a/topics/blockchain/assets/cross-chain-transfers-implementing-a-token-swap-from-base-chain-to-bitcoin-1.png and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_111fc823c12887002c2b8db6b1fb3bd1_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_111fc823c12887002c2b8db6b1fb3bd1_md5.webp deleted file mode 100644 index fa86e8ffc..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_111fc823c12887002c2b8db6b1fb3bd1_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_554108b34cb2175db1ecec15e3b7bfc3_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_554108b34cb2175db1ecec15e3b7bfc3_md5.webp deleted file mode 100644 index 67b8dada9..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_554108b34cb2175db1ecec15e3b7bfc3_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_5c893d5303e63e3b680f53b79b2878de_md5.svg b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_5c893d5303e63e3b680f53b79b2878de_md5.svg deleted file mode 100644 index 550efbf69..000000000 --- a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_5c893d5303e63e3b680f53b79b2878de_md5.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_63557ccfde06ef52ab5c8580590e6b8e_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_63557ccfde06ef52ab5c8580590e6b8e_md5.webp deleted file mode 100644 index e56687657..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_63557ccfde06ef52ab5c8580590e6b8e_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6966283d889117a7e021bfd7d29d47a7_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6966283d889117a7e021bfd7d29d47a7_md5.webp deleted file mode 100644 index 3da976576..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6966283d889117a7e021bfd7d29d47a7_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6d0dcbf83e5dea68e8d346c66b1637bb_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6d0dcbf83e5dea68e8d346c66b1637bb_md5.webp deleted file mode 100644 index 2a3f3148f..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_6d0dcbf83e5dea68e8d346c66b1637bb_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_cf2dd7dd7ccbdbdb75fb3d0f31ca5d68_md5.webp b/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_cf2dd7dd7ccbdbdb75fb3d0f31ca5d68_md5.webp deleted file mode 100644 index 41a8fa9ae..000000000 Binary files a/topics/blockchain/assets/design-system-for-layer-2-using-zk-rollup_cf2dd7dd7ccbdbdb75fb3d0f31ca5d68_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/distributed-systems_c7xyhh1.webp b/topics/blockchain/assets/distributed-systems_c7xyhh1.webp deleted file mode 100644 index 363e53bfd..000000000 Binary files a/topics/blockchain/assets/distributed-systems_c7xyhh1.webp and /dev/null differ diff --git a/topics/blockchain/assets/dollar-cost-averaging.webp b/topics/blockchain/assets/dollar-cost-averaging.webp deleted file mode 100644 index 6232c4c6c..000000000 Binary files a/topics/blockchain/assets/dollar-cost-averaging.webp and /dev/null differ diff --git a/topics/blockchain/assets/dollar-cost-averaging_1.webp b/topics/blockchain/assets/dollar-cost-averaging_1.webp deleted file mode 100644 index 7d88116a8..000000000 Binary files a/topics/blockchain/assets/dollar-cost-averaging_1.webp and /dev/null differ diff --git a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_2e60ff9cb71f5ecb312418e4228c7964_md5.webp b/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_2e60ff9cb71f5ecb312418e4228c7964_md5.webp deleted file mode 100644 index 19b612e4b..000000000 Binary files a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_2e60ff9cb71f5ecb312418e4228c7964_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_4717184a9972241d126ccab41d22390e_md5.webp b/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_4717184a9972241d126ccab41d22390e_md5.webp deleted file mode 100644 index 67d16af42..000000000 Binary files a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_4717184a9972241d126ccab41d22390e_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_871c33102b9552789598d25986ccd406_md5.webp b/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_871c33102b9552789598d25986ccd406_md5.webp deleted file mode 100644 index f6b5bddd2..000000000 Binary files a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_871c33102b9552789598d25986ccd406_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_a9852ce4f9889dd96d9efe61fd1cdc0c_md5.webp b/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_a9852ce4f9889dd96d9efe61fd1cdc0c_md5.webp deleted file mode 100644 index 5cb8856f6..000000000 Binary files a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_a9852ce4f9889dd96d9efe61fd1cdc0c_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_acecde099998e363519533076028fb4e_md5.webp b/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_acecde099998e363519533076028fb4e_md5.webp deleted file mode 100644 index fc5ac65d4..000000000 Binary files a/topics/blockchain/assets/fabric-hyperledger-architecture-explanation_acecde099998e363519533076028fb4e_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/liquidity-pool_pasted-image-20220322220453.webp b/topics/blockchain/assets/liquidity-pool_pasted-image-20220322220453.webp deleted file mode 100644 index 8c00d90b9..000000000 Binary files a/topics/blockchain/assets/liquidity-pool_pasted-image-20220322220453.webp and /dev/null differ diff --git a/topics/blockchain/assets/multisign-wallet-hero-image.webp b/topics/blockchain/assets/multisign-wallet-hero-image.webp deleted file mode 100644 index 8318bf1a1..000000000 Binary files a/topics/blockchain/assets/multisign-wallet-hero-image.webp and /dev/null differ diff --git a/topics/blockchain/assets/nft-fractionalization_y8tfcaq.webp b/topics/blockchain/assets/nft-fractionalization_y8tfcaq.webp deleted file mode 100644 index 0b9e27378..000000000 Binary files a/topics/blockchain/assets/nft-fractionalization_y8tfcaq.webp and /dev/null differ diff --git a/topics/blockchain/assets/plonky2_plonky-2-website.webp b/topics/blockchain/assets/plonky2_plonky-2-website.webp deleted file mode 100644 index 49111a128..000000000 Binary files a/topics/blockchain/assets/plonky2_plonky-2-website.webp and /dev/null differ diff --git a/topics/blockchain/assets/plonky2_tradeoffs.webp b/topics/blockchain/assets/plonky2_tradeoffs.webp deleted file mode 100644 index 6e5edfdf5..000000000 Binary files a/topics/blockchain/assets/plonky2_tradeoffs.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_1b54ce784c821f34b8d5d7218850095a84c9e054.webp b/topics/blockchain/assets/polygon-zkevm-architecture_1b54ce784c821f34b8d5d7218850095a84c9e054.webp deleted file mode 100644 index f5fa501d8..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_1b54ce784c821f34b8d5d7218850095a84c9e054.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_6066873078dcd11f9ef93601eba9237c52cbf11a.webp b/topics/blockchain/assets/polygon-zkevm-architecture_6066873078dcd11f9ef93601eba9237c52cbf11a.webp deleted file mode 100644 index 620d80b8d..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_6066873078dcd11f9ef93601eba9237c52cbf11a.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_fig-main-prts-zkpr.webp b/topics/blockchain/assets/polygon-zkevm-architecture_fig-main-prts-zkpr.webp deleted file mode 100644 index 3ebded4db..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_fig-main-prts-zkpr.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_fig3-zknode-arch-aa4d18996fba1849291ea18e3f11d955.webp b/topics/blockchain/assets/polygon-zkevm-architecture_fig3-zknode-arch-aa4d18996fba1849291ea18e3f11d955.webp deleted file mode 100644 index 2f498f74f..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_fig3-zknode-arch-aa4d18996fba1849291ea18e3f11d955.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zk-prover-design-approach.webp b/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zk-prover-design-approach.webp deleted file mode 100644 index ff9e0d9df..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zk-prover-design-approach.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkevm.webp b/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkevm.webp deleted file mode 100644 index f7682ce4e..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkevm.webp and /dev/null differ diff --git a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkprover.webp b/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkprover.webp deleted file mode 100644 index 70288584e..000000000 Binary files a/topics/blockchain/assets/polygon-zkevm-architecture_polygon-zkprover.webp and /dev/null differ diff --git a/topics/blockchain/assets/pos_jouvtnm.png.webp b/topics/blockchain/assets/pos_jouvtnm.png.webp deleted file mode 100644 index 774b0f3de..000000000 Binary files a/topics/blockchain/assets/pos_jouvtnm.png.webp and /dev/null differ diff --git a/topics/blockchain/assets/pos_xiuwh4m.png.webp b/topics/blockchain/assets/pos_xiuwh4m.png.webp deleted file mode 100644 index 51f6ef384..000000000 Binary files a/topics/blockchain/assets/pos_xiuwh4m.png.webp and /dev/null differ diff --git a/topics/blockchain/assets/starknet-architecture_starkdex.webp b/topics/blockchain/assets/starknet-architecture_starkdex.webp deleted file mode 100644 index b3f9da4aa..000000000 Binary files a/topics/blockchain/assets/starknet-architecture_starkdex.webp and /dev/null differ diff --git a/topics/blockchain/assets/starknet-architecture_starknet-current-architecture.webp b/topics/blockchain/assets/starknet-architecture_starknet-current-architecture.webp deleted file mode 100644 index e6155a1c2..000000000 Binary files a/topics/blockchain/assets/starknet-architecture_starknet-current-architecture.webp and /dev/null differ diff --git a/topics/blockchain/assets/starknet-architecture_starknet-l1l2.webp b/topics/blockchain/assets/starknet-architecture_starknet-l1l2.webp deleted file mode 100644 index 5cdbdaea8..000000000 Binary files a/topics/blockchain/assets/starknet-architecture_starknet-l1l2.webp and /dev/null differ diff --git a/topics/blockchain/assets/starknet-architecture_starknet-transactions-states.webp b/topics/blockchain/assets/starknet-architecture_starknet-transactions-states.webp deleted file mode 100644 index 9f98e6c74..000000000 Binary files a/topics/blockchain/assets/starknet-architecture_starknet-transactions-states.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_e_cute_routing.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_e_cute_routing.webp deleted file mode 100644 index 5cc7429bd..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_e_cute_routing.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-behavior.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-behavior.webp deleted file mode 100644 index 4f61abd83..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-behavior.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-model.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-model.webp deleted file mode 100644 index b9ce834c0..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton-actor-model.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_account_chain.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_account_chain.webp deleted file mode 100644 index 4af059235..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_account_chain.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_blockchain.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_blockchain.webp deleted file mode 100644 index 47bd1c1d1..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_blockchain.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_chain_of_txs.webp b/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_chain_of_txs.webp deleted file mode 100644 index 233bf6ff4..000000000 Binary files a/topics/blockchain/assets/ton_blockchain_of_blockchains_ton_chain_of_txs.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_core_concept_ton_bag_of_cells.webp b/topics/blockchain/assets/ton_core_concept_ton_bag_of_cells.webp deleted file mode 100644 index 1f7fe5afe..000000000 Binary files a/topics/blockchain/assets/ton_core_concept_ton_bag_of_cells.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_core_concept_ton_cell.webp b/topics/blockchain/assets/ton_core_concept_ton_cell.webp deleted file mode 100644 index c7a5a4447..000000000 Binary files a/topics/blockchain/assets/ton_core_concept_ton_cell.webp and /dev/null differ diff --git a/topics/blockchain/assets/ton_core_concept_ton_message_flow.webp b/topics/blockchain/assets/ton_core_concept_ton_message_flow.webp deleted file mode 100644 index 0291db35a..000000000 Binary files a/topics/blockchain/assets/ton_core_concept_ton_message_flow.webp and /dev/null differ diff --git a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_22ed3c2228f0f9355fcb48a2c63788ee_md5.webp b/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_22ed3c2228f0f9355fcb48a2c63788ee_md5.webp deleted file mode 100644 index e0d034877..000000000 Binary files a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_22ed3c2228f0f9355fcb48a2c63788ee_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_a5553a471de0e3b37e81f94efdf3f1c6_md5.webp b/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_a5553a471de0e3b37e81f94efdf3f1c6_md5.webp deleted file mode 100644 index a1c80c03e..000000000 Binary files a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_a5553a471de0e3b37e81f94efdf3f1c6_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_edacb4045a35d14a161b41e829079199_md5.webp b/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_edacb4045a35d14a161b41e829079199_md5.webp deleted file mode 100644 index d9312bf66..000000000 Binary files a/topics/blockchain/assets/using-foundry-for-evm-smart-contract-developement_edacb4045a35d14a161b41e829079199_md5.webp and /dev/null differ diff --git a/topics/blockchain/assets/web3-development-with-foundry-00.jpg b/topics/blockchain/assets/web3-development-with-foundry-00.jpg deleted file mode 100644 index 45f049f77..000000000 Binary files a/topics/blockchain/assets/web3-development-with-foundry-00.jpg and /dev/null differ diff --git a/topics/blockchain/assets/web3-development-with-foundry-01.jpg b/topics/blockchain/assets/web3-development-with-foundry-01.jpg deleted file mode 100644 index 99b598d52..000000000 Binary files a/topics/blockchain/assets/web3-development-with-foundry-01.jpg and /dev/null differ diff --git a/topics/blockchain/assets/web3-development-with-foundry-02.jpg b/topics/blockchain/assets/web3-development-with-foundry-02.jpg deleted file mode 100644 index 45c3cbd3c..000000000 Binary files a/topics/blockchain/assets/web3-development-with-foundry-02.jpg and /dev/null differ diff --git a/topics/blockchain/assets/web3-development-with-foundry-03.png b/topics/blockchain/assets/web3-development-with-foundry-03.png deleted file mode 100644 index 70fefc1bc..000000000 Binary files a/topics/blockchain/assets/web3-development-with-foundry-03.png and /dev/null differ diff --git a/topics/blockchain/blockchain-bridge.md b/topics/blockchain/blockchain-bridge.md deleted file mode 100644 index 529c5a583..000000000 --- a/topics/blockchain/blockchain-bridge.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: Blockchain bridge -description: A comprehensive guide to understanding blockchain bridges, their types, and how they facilitate communication and asset transfers between different blockchain networks. -date: 2022-06-21 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - blockchain-bridge - - web3 -redirect: - - /PJqkBw ---- - -![](assets/blockchain-bridge_blockruption-blockchain-300h.webp) - -Web3 or Dapp has evolved into an ecosystem of L1 blockchains and L2 scaling solutions, each designed with unique tradeoffs and processing capabilities. As the number of blockchain protocols or applications increases rapidly, so does the need to move assets across chains. To meet this need, we need communication channels between chains that are bridges. - -## What is a bridge? - -Blockchain bridges work just like a physical bridge connecting two physical locations, a blockchain bridge connecting two blockchain ecosystems. Bridges facilitate communication between blockchains through the transfer of message and assets. - -![](assets/blockchain-bridge.webp) - -## Why do we need bridges? - -All blockchains have their limits. In order for Ethereum to scale and keep up with demand, it needs to have rollouts. In addition, L1s such as Solana and Avalanche are designed differently to allow higher throughput but at the expense of decentralization. - -However, all blockchains operate in an isolated environment and have different rules and consensus mechanisms. This means that they cannot communicate natively and tokens cannot move freely between blockchains (Networks can only send messages in one direction so it will not be able to talk to the other network directly). Bridges born to connect blockchains, allowing for the trustless transfer of message and tokens between them. Bridges need an authentication mechanism, so there are different types of bridges. - -![](assets/blockchain-bridge_offchain-actors.webp) - -## Bridge use-case? - -- Transfer of assets and message across the chains -- Lower transaction fees -- Dapps on other blockchains -- Explore the blockchain ecosystem, users access new platforms and take advantage of different chains. -- Developers from different blockchain ecosystems to collaborate and build new platforms for users. -- Own natural crypto assets - -![](assets/blockchain-bridge_bridge-use-cases.webp) - -## How does the bridge work? - -Basically a bridge works between 2 networks by listening for events arising from one network and forwarding information to the other network. So the basic problem is in the mechanism to ensure safety when forwarding information as well as message authentication and message monitoring. - -### What are bridge components? - -- Monitor: There is usually an actor, either a “oracle”, a “validator” or a “relayer”, that monitors the state on the source chain. -- Message Passing/Forwarding: After an agent selects an event, it needs to pass information from the source chain to the destination chain. -- Consensus: In some models, consensus is required between the parties monitoring the source chain to forward that information to the destination chain. -- Signing: Agents need to cryptographically sign, individually or as part of a threshold signature scheme, the information sent to the destination chain. - -## Classification of bridges - -![](assets/blockchain-bridge_classify-bridge.webp) - -### External links & validators - -There is usually a group of validators that monitor the "mailbox" address on the source chain, by consensus perform an action on the destination chain. Content transfer is usually done by locking the message in the mailbox and generating an equivalent message on the destination thread. These are usually bonded validators with a separate token as the security model. - -![](assets/blockchain-bridge-external-links.webp) - -### Lightweight client & relays - -Agents monitor events on the source chain and generate cryptographic proof of past events that have been recorded on that chain. These proofs are then relayed, along with block headers, to contracts (i.e. "clients") on the target chain, which then verify that a given event was logged and executed. perform an action after that verification. There is a requirement for some actors to "forward" block headers and proofs. Although users can “self-forward” transactions, there is a realistic assumption that forwarders will continuously forward data. This is a relatively secure bridge design because it ensures trustless validating distribution without trusting intermediate entities, but it is also resource intensive because developers have to build build a new smart contract on each new destination chain that parses the proofs of state from the source chain and the confirmation itself is a lot of gas. - -![](assets/blockchain-bridge_lightweight-client-&-relays.webp) - -### Liquidity network - -This is similar to a peer-to-peer network where each node acts as a “router” containing a “store” of assets of both the source and destination chains. These networks typically leverage the security of the underlying blockchain; Through the use of locking and contention mechanisms, users are assured that routers cannot run away with user funds. As a result, liquidity networks like Connext may be a safer option for users who are transferring large amounts of value. Furthermore, this type of bridge may be most suitable for cross-chain asset transfers because the assets provided by the router are the origin of the destination chain and not a derivative, which are not fully interchangeable. - -## Depending on the design, each bridge will have its own characteristics - -Security: Assumptions about reliability and viability, tolerance to malicious actors, safety of user funds, and reflectiveness. - -- Speed: Latency to complete the transaction, as well as ensuring finality. There is usually a trade-off between speed and security. -- Connectivity: Choice of target chain for both users and developers, as well as different difficulty levels to integrate one more target chain. -- Efficient use of capital: Economics revolves around the capital needed to secure the system and the transaction costs to transfer assets. -- Authenticity: Ability to transfer specific assets, more complex state, and/or make cross-chain contract calls. - -## Interoperability dilemma - -![](assets/blockchain-bridge_interoperability-dilemma.webp) - -Similar to Trilemma in terms of scalability, there exists a Trilemma of interoperability in the Ethereum ecosystem. The Interop protocol can only have two of the following three properties: - -- Untrusted: has the same security as basic domains. -- Scalability: can be supported on any domain. -- Generalizability: capable of handling arbitrary cross-domain data. - -## Risks of using bridge - -- There is a bug in the smart contract. -- The underlying blockchain is hacked or the block is rolledback: The data of a block is preserved in one chain but cannot be changed in the other chain. -- Bridge moderators have malicious intent in a trusted bridge. -- Hacked bridge: attack via internal consensus mechanism. -- The user makes a mistake when manipulating. -- Congested or hacked chains will affect bridging. - -![](assets/blockchain-bridge_lock,-mint-and-burn.webp) - -One recent hack was Solana's Wormhole Bridge, where 120k wETH ($325 million USD) was stolen in the hack. And Vitalik himself must have a reputation for the lack of safety of the bridges. - -- [The multiple million exploit](https://decrypt.co/76117/thorchains-rune-token-slides-following-multi-million-exploit) -- [Thorchain hacks](https://www.coindesk.com/markets/2021/07/23/blockchain-protocol-thorchain-suffers-8m-hack/). -- [The PolyNetwork hack](https://edition.cnn.com/2021/08/11/tech/crypto-hack/index.html). - -## Case study - -- One of our product - [icrosschain.io](https://icrosschain.io/) -- Wormhole -- Thorchain - -## Reference - -- https://en.wikipedia.org/wiki/Ethereum -- [What are blockchain bridges and how can we classify them?](https://blog.li.finance/what-are-blockchain-bridges-and-how-can-we-classify-them-560dc6ec05fa) Feb 18, 2021 - Arjun Chand -- [Multichain users lose (https://cryptobriefing.com/multichain-users-lose-1-4m-due-bridge-bug/).4M due to bridge bug](https://cryptobriefing.com/multichain-users-lose-1-4m-due-bridge-bug/) -- [Vitalik Buterin skeptical of cross-chain bridges](https://cryptobriefing.com/vitalik-buterin-skeptical-of-cross-chain-bridges/) -- [Latest DeFi bridge exploit results in $4.4M losses for Meter](https://cointelegraph.com/news/latest-defi-bridge-exploit-results-in-4-4m-losses-for-meter) diff --git a/topics/blockchain/blockchain-oracle.md b/topics/blockchain/blockchain-oracle.md deleted file mode 100644 index f3b251171..000000000 --- a/topics/blockchain/blockchain-oracle.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: Blockchain oracle -short_title: ¶ Oracle -description: Learn how blockchain oracles solve the oracle problem by connecting smart contracts with real-world data, enabling secure input, output, cross-chain communication, and off-chain computing. -date: 2022-03-17 -authors: - - trankhacvy -github_id: trankhacvy -tags: - - blockchain - - oracle -redirect: - - /4rTExA ---- - -## The blockchain oracle problem - -Blockchains have a fundamental limitation: they cannot natively communicate with systems of the outside world. This lack of external connectivity, known as "the oracle problem", prevents smart contracts from verifying external events, trigger actions on existing systems, and providing users the full range of functionality. - -## What is a blockchain oracle? - -A blockchain oracle is a third-party service that connects smart contracts with the outside world, primarily to feed information in about the world around, but the reverse is also true. - -![](assets/blockchain-oracle_ins_and_outs_of_the_blockchain_ecosystem.webp) _Blockchain oracles connect blockchains to inputs and outputs of the real world (Image source: [Chainlink](https://chain.link/))_ - -## Types of blockchain oracles - -### Input oracles - -The most widely recognized type of oracle today is known as an “input oracle”, which fetches data from the real-world(off-chain) and delivers it onto a blockchain network for smart contracts consumption. A good example of this are the Chainlink Price Feeds. - -### Output oracles - -The opposite of input oracles are "output oracles", which allow smart contracts to send commands to off-chain systems to trigger and execute certain actions. - -### Cross-chain oracles - -This type of oracle can read and write information between different blockchains. Cross-chain oracles enable interoperability for moving both data and assets between blockchains. - -### Compute-enabled oracles - -This type of oracle provides decentralized services with secure, off-chain computation that would otherwise be impractical to do on the blockchain due to technical, legal, or financial constraints. - -## Notable blockchain oracles - -- Chainlink (LINK) -- Band Protocol (BAND) -- Teller (TRB) -- Decentralized Information Asset (DIA) -- API3 - -## Reference - -- [Wikipedia - Blockchain oracle](https://en.wikipedia.org/wiki/Blockchain_oracle#:~:text=A%20blockchain%20oracle%20is%20a,that%20decentralised%20knowledge%20is%20obtained.) -- [https://chain.link/education/blockchain-oracles](https://chain.link/education/blockchain-oracles) -- [https://coin98.net/what-is-blockchain-oracle](https://coin98.net/what-is-blockchain-oracle) diff --git a/topics/blockchain/blockchain-security.md b/topics/blockchain/blockchain-security.md deleted file mode 100644 index 989e98285..000000000 --- a/topics/blockchain/blockchain-security.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Blockchain security checklist -description: Ensure blockchain security with this checklist covering centralization, event logging, code safety, gas optimization, unit testing, coding style, logical issues, and potential attacks for safer smart contract development. -date: null -redirect: - - /2rZnAA ---- - -- [ ] Understanding the project - -##### Centralization / privilege - -- [ ] Secure the function have access by owner -- [ ] Initial token distribution, should prefix receiver address -- [ ] Use multisign wallet for dev -- [ ] Unrestricted privilege function - -##### Event log - -- [ ] Event for any set function -- [ ] Event for significant transactions -- [ ] Event name must clear and avoid misunderstand -- [ ] Favor capitalization and a prefix in front of events (we suggest Log) - -##### Volatile code - -- [ ] Avoid Reentrancy -- [ ] Check the return value of external call such as transfer -- [ ] Avoid state change after call external function -- [ ] Math Operation -- [ ] Check insecure arithmetic, integer under/overflow -- [ ] Validity check -- [ ] Clear Code structure -- [ ] Don't use transfer() or send() -- [ ] Handle errors in external call ( can use try/catch) -- [ ] Favor pull over push for external calls -- [ ] Don't delegatecall to untrusted code -- [ ] Force-feeding Ether - -##### Gas optimization - -- [ ] Don't use recursive function -- [ ] Use appropriate type (uint8, map...) -- [ ] Initial variable in constructor -- [ ] Use external function instead of public - -##### Unit test - -- [ ] Unit test for get/set function -- [ ] Unit test for overflow data -- [ ] Unit test for external call -- [ ] Run unit test before deploy - -##### Coding style - -- [ ] Language specific -- [ ] Store config of upgrade contract and push to git -- [ ] Check Reusable Code, use modifier -- [ ] Check for minimal source code -- [ ] Have note/status for deployed code - -##### Logical issue - -- [ ] Check over minted token -- [ ] Don't trust tx.origin for authorization, use msg.sender for authorization. -- [ ] Check timestamp for logic will be manipulated by a miner -- [ ] Avoid using block.number as a timestamp - -##### Potential attack - -- [ ] Check for Potential Sandwich Attack diff --git a/topics/blockchain/blockchain-simplified.md b/topics/blockchain/blockchain-simplified.md deleted file mode 100644 index 317a2d91d..000000000 --- a/topics/blockchain/blockchain-simplified.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: "Blockchain fundamentals" -description: "Blockchain breaks down the complex technology behind Bitcoin and other cryptocurrencies into simple terms anyone can understand. Learn how distributed ledgers work, why they're secure, and what makes them revolutionary for digital trust." -date: 2020-12-19 -authors: - - tieubao -tags: - - blockchain - - cryptocurrency - - distributed-systems - - security - - bitcoin ---- - -# Blockchain fundamentals - -Think of blockchain as a shared notebook that everyone in a network keeps a copy of. When someone writes in it, everyone else updates their copy too. This makes it nearly impossible to cheat because you'd need to change every single notebook at the same time. - -That's blockchain in its simplest form. It's the technology powering Bitcoin, Ethereum, and other cryptocurrencies, but the concept goes way beyond digital money. - -## How blockchain actually works - -**The building blocks** - -Every blockchain is made of "blocks" (hence the name). Each block is like a page in that shared notebook, containing: - -- **Transaction records**: Who sent what to whom ("Alice sends Bob 1 Bitcoin") -- **Timestamp**: When this happened -- **Hash**: A unique fingerprint for this block's data -- **Previous block's hash**: Links to the block before it, creating the chain - -The hash is crucial. It's a unique number generated from the block's data. Change even one character in the block, and you get a completely different hash. This makes tampering obvious. - -**The mining process** - -Here's where it gets interesting. When you make a transaction, it doesn't immediately go into the blockchain. Instead: - -1. Your transaction gets broadcast to the network -2. It sits in a pool of "pending" transactions -3. Miners collect these transactions into a new block -4. They compete to solve a computational puzzle to add this block -5. The first to solve it wins, adds the block, and earns a reward - -This puzzle is intentionally difficult. Miners have to try millions of random numbers (called a "nonce") until they find one that makes the block's hash start with a specific number of zeros. It's like a lottery where computing power buys you more tickets. - -**Network consensus** - -Once a miner solves the puzzle, they broadcast the new block. Other nodes verify it in seconds (solving is hard, checking is easy), then add it to their copy of the blockchain. This is how thousands of computers stay synchronized without a central authority. - -## Why blockchain is secure - -The security comes from three key features: - -**Distributed copies**: Every participant has the complete blockchain. To successfully cheat, you'd need to control over 50% of all nodes, which becomes practically impossible as the network grows. - -**Cryptographic hashing**: Each block contains the previous block's hash. Change one block, and you break the chain for all subsequent blocks. An attacker would need to redo the computational work for every block that follows. - -**Proof of work**: The mining process requires real computational effort. Creating fake blocks is expensive, while verifying legitimate ones is cheap. This economic incentive keeps the network honest. - -## Solving the "two blocks at once" problem - -Sometimes two miners solve the puzzle simultaneously, creating competing versions of the blockchain. The network handles this elegantly: - -- Both blocks get accepted temporarily -- The network follows the "longest chain rule" -- Whichever branch gets the next block first becomes the official version -- The other branch gets discarded - -This happens naturally because the longest chain represents the most computational work, making it the most trusted version. - -## Making it practical for everyday use - -Not everyone needs to download the entire blockchain (Bitcoin's is hundreds of gigabytes). "Lightweight" clients can verify transactions by checking just the relevant parts, while full nodes maintain the complete ledger. - -This makes blockchain accessible to regular users without requiring massive storage or computing power. - -## Real-world applications - -**Beyond cryptocurrency** - -While Bitcoin popularized blockchain, the technology works for any data that needs to be: - -- Tamper-proof -- Transparent -- Decentralized - -**Practical examples:** - -- **Supply chain tracking**: Following products from factory to store -- **Digital identity**: Secure, self-controlled personal records -- **Smart contracts**: Automated agreements that execute themselves -- **Voting systems**: Transparent, verifiable elections - -## The trade-offs to consider - -Blockchain isn't perfect. The security and decentralization come with costs: - -- **Energy consumption**: Bitcoin mining uses massive amounts of electricity -- **Speed limitations**: Traditional blockchains process fewer transactions per second than credit card networks -- **Scalability challenges**: As more people join, the system can become slower and more expensive - -Newer blockchain designs are addressing these issues with different consensus mechanisms and scaling solutions. - -## Building with blockchain - -For developers curious about implementation, a basic blockchain involves: - -- **Transaction signing**: Using private keys to prove ownership -- **Peer-to-peer networking**: Sharing transactions across nodes -- **Block creation**: Bundling transactions with proof of work -- **Chain validation**: Following the longest valid chain rule - -You could prototype a simple version in about 100 lines of code, though production systems require much more sophistication for security and performance. - -## Why blockchain matters - -Blockchain's real innovation isn't the technology itself, it's what it enables: **digital trust without intermediaries**. For the first time, strangers can transact directly with confidence, knowing the system prevents fraud and maintains accurate records. - -This removes the need for banks, governments, or other middlemen in many situations, potentially reducing costs and increasing accessibility. Whether you're sending money across borders, proving ownership of digital assets, or creating transparent voting systems, blockchain provides a foundation for trustworthy interactions in our digital world. - -The technology is still evolving, but the core concept of shared, tamper-proof ledgers is already changing how we think about digital trust and value exchange. diff --git a/topics/blockchain/documents.md b/topics/blockchain/documents.md deleted file mode 100644 index 53788c8fb..000000000 --- a/topics/blockchain/documents.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: Documents -description: Learn how to properly document smart contracts with contact info, testing history, known issues, procedures, specifications, and deployment status to ensure security and transparency. -date: null -redirect: - - /uGwgQA ---- - -When launching a contract that will have substantial funds or is required to be mission critical, it is important to include proper documentation. - -## Contact - -- Who to contact with issues -- Names of programmers and/or other important parties -- Chat room where questions can be asked - -## History - -- Testing (including usage stats, discovered bugs, length of testing) -- People who have reviewed code (and their key feedback) - -## Known-issues - -- Key risks with contract - - e.g., You can lose all your money, hacker can vote for certain outcomes -- All known bugs/limitations -- Potential attacks and mitigants -- Potential conflicts of interest (e.g., will be using yourself, like Slock.it did with the DAO) - -## Procedures - -- Action plan in case a bug is discovered (e.g., emergency options, public notification process, etc.) -- Wind down process if something goes wrong (e.g., funders will get percentage of your balance before attack, from remaining funds) -- Responsible disclosure policy (e.g., where to report bugs found, the rules of any bug bounty program) -- Recourse in case of failure (e.g., insurance, penalty fund, no recourse) - -## Specification - -- Specs, diagrams, state machines, models, and other documentation that helps auditors, reviewers, and the community understand what the system is intended to do. -- Many bugs can be found just from the specifications, and they are the least costly to fix. -- Rollout plans that include details listed [here](precautions.md), and target dates. - -## Status - -- Where current code is deployed -- Compiler version, flags used, and steps for verifying the deployed bytecode matches the source code -- Compiler versions and flags that will be used for the different phases of rollout. -- Current status of deployed code (including outstanding issues, performance stats, etc.) diff --git a/topics/blockchain/dollar-cost-averaging.md b/topics/blockchain/dollar-cost-averaging.md deleted file mode 100644 index 4d60f4fd2..000000000 --- a/topics/blockchain/dollar-cost-averaging.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -title: Dollar Cost Averaging (DCA) -description: Dollar-Cost Averaging (DCA) is a simple yet powerful investment strategy that can help you minimize risk and maximize long-term returns. With DCA, you invest a fixed amount of money into an asset (crypto, stocks, funds, etc.) at regular intervals, regardless of the current market price. The goal is to accumulate as much of the asset as possible over time. This strategy is suitable for all investors, especially beginners. -date: 2024-05-03 -authors: - - bringastar -tags: - - blockchain - - crypto -redirect: - - /YMoOqQ ---- - -Hey peeps! Ever heard of DCA (Dollar-Cost Averaging)? It's like the hottest strategy in town, and for good reason - it can seriously boost your chances of crypto success. This article is your one-stop shop for all things DCA. I'll break down what it is, the different ways you can do it, and some mistakes to avoid so you can DCA like a pro. - -![](assets/dollar-cost-averaging.webp) - -## What is the deal with DCA? - -Dollar-Cost Averaging (DCA) is a simple yet powerful investment strategy that can help you minimize risk and maximize long-term returns. With DCA, you **invest a fixed amount of money into an asset** (crypto, stocks, funds, etc.) **at regular intervals, regardless of the current market price**. The goal is to accumulate as much of the asset as possible over time. This strategy is suitable for all investors, especially beginners. - -In simpler terms, it's about throwing some money into crypto at regular intervals, no matter if the price is mooning or going down. This way, you average out the cost over time and snag some sweet profits in the long run. - -While not a new concept, DCA has been proven effective for decades in traditional markets like stocks and gold. In the high-potential crypto market, DCA has become even more popular. The mentality of "buying more when the price is down" is very common among crypto enthusiasts, especially when the market has been skyrocketing in recent years. While "bottom fishing" can be tempting, DCA is the key to safe and effective investing in a volatile market like crypto. - -## How DCA works - -To better understand how this investment strategy works, let's look at an example: -Suppose an investor decides to invest $3,000 over 6 months by buying $1,000 of a specific asset for the first **3 months** and applying the DCA strategy. However, if you want to see the powerful impact of cost averaging, check out the difference between cost averaging and not using the strategy (when you invest the entire $3,000 in month 1). - -The scenario would be as follows: - -![](assets/dollar-cost-averaging_1.webp) - -From March onwards, you can see that DCA has increased in value compared to not having a strategy. **This is because the more money you invest at once, the more risk you are exposed to market fluctuations.** - -- When investing without a strategy, the $3,000 is subject to the entire decline of March + April and has to take the rest of the time to recover. -- With the DCA strategy, you can minimize the loss by 20% and take advantage of it to catch the bottom and collect more assets. - -To implement DCA, follow these steps: - -1. Choose the right investment asset for DCA -1. Decide on the investment amount per period -1. Identify your maximum acceptable loss -1. Choose an investment frequency (weekly, monthly,…) -1. Define entry, profit-taking, and stop-loss points - -> **Note**: While DCA can be a good strategy, there can be more transaction fees compared to a one-time investment. - -## DCA in practice - -In theory, DCA helps you buy an asset at different prices, thereby minimizing the risk of market fluctuations. However, in practice, traders and investors apply DCA in many different ways, specifically as follows: - -**DCA in a Bull Market** - -This strategy is a way of investing in a rising market by buying a fixed amount of an asset at regular intervals, such as every month. This means that you will buy more of the asset when the price is low and less of the asset when the price is high. The average purchase price will increase over time. - -- **Pros:** Buy with the mindset of "making a profit". You aim to buy low and sell high for maximum profit. While buying at higher prices increases your average cost, there's still room for profit if the price keeps rising. This approach lets you potentially maximize your gains. -- **Cons:** Buy more at higher prices can hurt your profit potential. Remember, the crypto market is known for big, unexpected crashes, especially after periods of strong price increases. - -Since 2021, this has perhaps been the strategy adopted by large investment funds, such as MicroStrategy, BlackRock, and Fidelity, if you often follow articles on the allocation of capital flows from investment funds. - -**DCA in a Bear Market** - -DCA in a bear market is the opposite of DCA in a bull market. When falling market, investors buy more of an asset when the price is low and less when the price is high. The average purchase price will decrease over time. - -- **Pros**: You can buy assets at lower prices, which could mean higher profits when the market recovers. And regular investing helps you avoid trying to time the market, which can be risky. -- **Cons**: The risk of this method is that hard to know when to buy. It's tough to know when the market will hit its lowest point, so you might buy too early or too late. Prices could keep falling, even if you're buying regularly. - -[El Salvador has been consistently "buying the dip"](https://www.thestreet.com/investing/el-salvador-buys-the-dip-again#:~:text=At%20500%20coins%2C%20this%20purchase,150%20bought%20in%20September%202021.) in Bitcoin throughout 2021-2022, driven by the belief of its leaders in Bitcoin's growth. CoinDesk estimates that El Salvador currently holds 2,381 Bitcoin in its treasury, worth a total of $147 million. Most recently, the country said that if it sold all of its Bitcoin now, it would make a profit of over 40%, or $41.6 million. - -**DCA for the Long Haul** - -A simple way to invest that's perfect for people who want to put their money away for the long term and don't want to worry too much about the ups and downs of the market. It's like putting money in a savings account, but instead of just earning interest, you're also buying shares of an asset, like stocks or cryptocurrency. - -- **Pros**: Don't need to be an expert to use this strategy. You can start investing with even a small amount of money. The market has always gone up in the long run, so this strategy can be a good way to grow your wealth over time. -- **Cons**: The price of your investment could go down in the short term, so you could lose money. If you lack knowledge and experience, you will be easily scared by price changes. This can cause you to sell your investments when the market goes down. - -Crypto prices can go up and down quickly, making this strategy tricky to use. Still, some traders have pulled it off in the short term to buy at the best prices. - -**Flexible DCA** - -The flexible DCA strategy takes regular DCA to the next level. You can adjust your investment amount based on the market's ups and downs. So, if the market is falling, you can invest a little more to buy more when prices are low. And when the market is rising, you can invest a little less to avoid buying at high prices. - -- **Pros**: You can change how much you invest based on the market and your own finances. This strategy can work for both beginners and experienced investors. -- **Cons**: To make the most of this strategy, you should have some basic understanding of the financial markets. - -**Comparison table:** -| Bull Market | Bear Market | Long Haul | Flexible | -| --- | --- | --- | --- | -| Invest in a rising market | Invest in a falling market | Believe that the market will always trend upwards in the long run | Invest based on the market -| Buy a fixed amount of it at **regular intervals** (daily, weekly, monthly,…) | Buy **more of an asset** when the price is low and less when the price is high | Buy a fixed amount of it over a **period** of 3-5 years | Adjust your amount based on the market's ups and downs -| The average purchase price will **increase** over time | The average purchase price will **decrease** over time | Invest without worrying about market ups and downs | The average purchase price will **increase** or **decrease** gradually depending on market trends | - -### Can DCA be applied to anything? - -Hell no! While DCA is a simple and easy-to-understand strategy, it's not a magic bullet for beating the market. Many investors combine DCA with portfolio diversification to reduce risk. - -**DCA is best suited for low-risk assets such as:** - -- Bitcoin -- Ethereum -- S&P 500 Index Fund -- ETFs -- Retirement funds (e.g., 401(k)) - -**Common mistakes when using DCA:** - -- **Leverage trading:** Leverage trading (such as margin, futures, etc.) is already inherently risky, as traders use more money than they have. Combining it with DCA amplifies this risk. It's recommended to only use DCA for spot trading. -- **Low-liquidity altcoins:** Even Bitcoin, with its high trading volume, experiences significant price fluctuations. Low-liquidity altcoins pose even greater risks. These altcoins may be poorly designed or pump-and-dump schemes, requiring thorough research before investing. -- **Altcoin/BTC pairs:** While experienced traders may profit from this approach, altcoin/BTC pairs expose you to two layers of volatility: altcoin/BTC pair fluctuations and BTC/USD fluctuations. This can lead to double losses if you use your losing BTC to buy a falling altcoin/BTC. -- **Meme coins:** Meme coins are essentially overpriced jokes. View them as a portfolio diversification tool rather than a DCA strategy target. - -Bear in mind that before you start DCAing during a downturn, ask yourself: "How much risk can I handle?" Only DCA with money you can afford to lose, and be prepared for the possibility that some of it might not come back. - -Overall, DCA is a long-term investment strategy. Don't get discouraged by short-term market fluctuations. Stay disciplined and consistent with your investments, and you'll be well on your way to achieving your financial goals. diff --git a/topics/blockchain/external-calls.md b/topics/blockchain/external-calls.md deleted file mode 100644 index cc4271d88..000000000 --- a/topics/blockchain/external-calls.md +++ /dev/null @@ -1,220 +0,0 @@ ---- -title: External calls -description: Learn best practices for safe external calls in Solidity, including avoiding reentrancy, handling errors, using call() over transfer(), and never delegatecall untrusted contracts to secure smart contracts. -date: null -redirect: - - /2m57IQ ---- - -#### Use caution when making external calls - -Calls to untrusted contracts can introduce several unexpected risks or errors. External calls may -execute malicious code in that contract _or_ any other contract that it depends upon. As such, -every external call should be treated as a potential security risk. When it is not possible, or -undesirable to remove external calls, use the recommendations in the rest of this section to -minimize the danger. - ---- - -#### Mark untrusted contracts - -When interacting with external contracts, name your variables, methods, and contract interfaces in -a way that makes it clear that interacting with them is potentially unsafe. This applies to your -own functions that call external contracts. - -```sol -// bad -Bank.withdraw(100); // Unclear whether trusted or untrusted - -function makeWithdrawal(uint amount) { // Isn't clear that this function is potentially unsafe - Bank.withdraw(amount); -} - -// good -UntrustedBank.withdraw(100); // untrusted external call -TrustedBank.withdraw(100); // external but trusted bank contract maintained by XYZ Corp - -function makeUntrustedWithdrawal(uint amount) { - UntrustedBank.withdraw(amount); -} -``` - ---- - -#### Avoid state changes after external calls - -Whether using _raw calls_ (of the form `someAddress.call()`) or _contract calls_ (of the form -`ExternalContract.someMethod()`), assume that malicious code might execute. Even if -`ExternalContract` is not malicious, malicious code can be executed by any contracts _it_ calls. - -One particular danger is malicious code may hijack the control flow, leading to vulnerabilities due -to reentrancy. - -If you are making a call to an untrusted external contract, _avoid state changes after the call_. -This pattern is also sometimes known as the -[checks-effects-interactions pattern](http://solidity.readthedocs.io/en/develop/security-considerations.html?highlight=check%20effects#use-the-checks-effects-interactions-pattern). - -See [SWC-107](https://swcregistry.io/docs/SWC-107) - ---- - -#### Don't use `transfer()` or `send()`. - -`.transfer()` and `.send()` forward exactly 2,300 gas to the recipient. The goal of this hardcoded -gas stipend was to prevent, but this only -makes sense under the assumption that gas costs are constant. Recently -[EIP 1884](https://eips.ethereum.org/EIPS/eip-1884) was included in the Istanbul hard fork. One of -the changes included in EIP 1884 is an increase to the gas cost of the `SLOAD` operation, causing a -contract's fallback function to cost more than 2300 gas. - -It's recommended to stop using `.transfer()` and `.send()` and instead use `.call()`. - -``` -// bad -contract Vulnerable { - function withdraw(uint256 amount) external { - // This forwards 2300 gas, which may not be enough if the recipient - // is a contract and gas costs change. - msg.sender.transfer(amount); - } -} - -// good -contract Fixed { - function withdraw(uint256 amount) external { - // This forwards all available gas. Be sure to check the return value! - (bool success, ) = msg.sender.call.value(amount)(""); - require(success, "Transfer failed."); - } -} -``` - -Note that `.call()` does nothing to mitigate reentrancy attacks, so other precautions must be -taken. To prevent reentrancy attacks, it is recommended that you use the -[checks-effects-interactions pattern](https://solidity.readthedocs.io/en/develop/security-considerations.html?highlight=check%20effects#use-the-checks-effects-interactions-pattern). - ---- - -#### Handle errors in external calls - -Solidity offers low-level call methods that work on raw addresses: `address.call()`, -`address.callcode()`, `address.delegatecall()`, and `address.send()`. These low-level methods never -throw an exception, but will return `false` if the call encounters an exception. On the other hand, -_contract calls_ (e.g., `ExternalContract.doSomething()`) will automatically propagate a throw (for -example, `ExternalContract.doSomething()` will also `throw` if `doSomething()` throws). - -If you choose to use the low-level call methods, make sure to handle the possibility that the call -will fail, by checking the return value. - -```sol -// bad -someAddress.send(55); -someAddress.call.value(55)(""); // this is doubly dangerous, as it will forward all remaining gas and doesn't check for result -someAddress.call.value(100)(bytes4(sha3("deposit()"))); // if deposit throws an exception, the raw call() will only return false and transaction will NOT be reverted - -// good -(bool success, ) = someAddress.call.value(55)(""); -if(!success) { - // handle failure code -} - -ExternalContract(someAddress).deposit.value(100)(); -``` - -See [SWC-104](https://swcregistry.io/docs/SWC-104) - ---- - -#### Favor _pull_ over _push_ for external calls - -External calls can fail accidentally or deliberately. To minimize the damage caused by such -failures, it is often better to isolate each external call into its own transaction that can be -initiated by the recipient of the call. This is especially relevant for payments, where it is -better to let users withdraw funds rather than push funds to them automatically. (This also reduces -the chance of problems with gas limits.) Avoid -combining multiple ether transfers in a single transaction. - -```sol -// bad -contract auction { - address highestBidder; - uint highestBid; - - function bid() payable { - require(msg.value >= highestBid); - - if (highestBidder != address(0)) { - (bool success, ) = highestBidder.call.value(highestBid)(""); - require(success); // if this call consistently fails, no one else can bid - } - - highestBidder = msg.sender; - highestBid = msg.value; - } -} - -// good -contract auction { - address highestBidder; - uint highestBid; - mapping(address => uint) refunds; - - function bid() payable external { - require(msg.value >= highestBid); - - if (highestBidder != address(0)) { - refunds[highestBidder] += highestBid; // record the refund that this user can claim - } - - highestBidder = msg.sender; - highestBid = msg.value; - } - - function withdrawRefund() external { - uint refund = refunds[msg.sender]; - refunds[msg.sender] = 0; - (bool success, ) = msg.sender.call.value(refund)(""); - require(success); - } -} -``` - -See [SWC-128](https://swcregistry.io/docs/SWC-128) - ---- - -#### Don't delegatecall to untrusted code - -The `delegatecall` function is used to call functions from other contracts as if they belong to the -caller contract. Thus the callee may change the state of the calling address. This may be insecure. -An example below shows how using `delegatecall` can lead to the destruction of the contract and -loss of its balance. - -```sol -contract Destructor -{ - function doWork() external - { - selfdestruct(0); - } -} - -contract Worker -{ - function doWork(address _internalWorker) public - { - // unsafe - _internalWorker.delegatecall(bytes4(keccak256("doWork()"))); - } -} -``` - -If `Worker.doWork()` is called with the address of the deployed `Destructor` contract as an -argument, the `Worker` contract will self-destruct. Delegate execution only to trusted contracts, -and **never to a user supplied address**. - -!!! Warning - -- Don't assume contracts are created with zero balance An attacker can send ether to the address of a contract before it is created. Contracts should not assume that their initial state contains a zero balance. -- See [issue 61](https://github.com/ConsenSys/smart-contract-best-practices/issues/61) for more details. -- See [SWC-112](https://swcregistry.io/docs/SWC-112) diff --git a/topics/blockchain/fabric-hyperledger-architecture-explanation.md b/topics/blockchain/fabric-hyperledger-architecture-explanation.md deleted file mode 100644 index 5f38b2c50..000000000 --- a/topics/blockchain/fabric-hyperledger-architecture-explanation.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: Fabric Hyperledger architecture explanation -description: "Learn how Hyperledger's open source blockchain frameworks and tools enable secure, permissioned enterprise networks with smart contracts, distributed ledgers, and flexible governance models." -date: 2019-05-15 -tags: - - architecture - - blockchain -redirect: - - /kAJQFA ---- - -## What is Hyperledger - -Hyperledger is an open source community focused on developing a suite of stable frameworks, tools and libraries for enterprise-grade blockchain deployments. - -It serves as a neutral home for various distributed ledger frameworks including Hyperledger Fabric, Sawtooth, Indy, as well as tools like Hyperledger Caliper and libraries like Hyperledger Ursa. - -## Key concept of blockchain in Hyperledger - -### A distributed ledger - -Records all the transactions that take place on the network - -Replicated across many network participants, each of whom collaborate in its maintenance - -### Smart contracts - -Self-executing contract if match pre-defined rules - -### Consensus - -- Consensus problem( A fundamental problem in distributed-computing and multi-agent systems is to achieve overall system reliability in the presence of a number of faulty processes by agrees some data value that is needed during computation) -- Append value through transaction to network -- Update only when transactions are approved by the appropriate participants -- Update with the same order in any participants - -## Hyperledger Fabric models - -### Assets - -Asset definitions enable the exchange of almost anything with monetary value over the network as a collection of key-value pairs. - -### Chaincode - -Enforces the rules for defining, modifying assets, it's the business logic. Execute against the ledger's current state database and initiated through a transaction proposal -> results in a set of key-value writes that can be submitted to the network and applied to the ledger on all peers. - -### Ledger - -Ledger is the sequenced, tamper-resistant record of all state transactions in the fabric. There is one ledger per channel, each peer maintains a copy of the ledger of their channels. - -### Permissioned vs permissionless - -**Similarities** -Both are unalterable digitally signed ledgers which are distributed through peer-to-peer network Both maintain ledgers which are updated through a protocol named as consensus. Both claim to maintain an immutable ledger - -**Differences** - -1. Permissioned - -- Varying decentralization: members of the blockchain network are free to negotiate and come to a decision concerning the level of decentralization that the network will have -- Close: consortia members have the ability to restrict access. Not required to be transparent, members can choose to do so freely, depending on the inner organization of the businesses. -- Clean Governance structure: governance is decided by members of the business network. Decisions are made on a central level, where the entirety of the network must agree to a change - -→ More control on the central level, more private -> use for B2B business models - -2. Permissionless - -- Decentralized: no central entity has the authority to edit the ledger, shut down the network or change protocols. -- Open: anyone can access, participate in the validation process -- Public: anything running is verifiable on the networks by everybody, running full nodes, store full history of all transactions -- Anonymity: members join the network as fully anonymous, networks don't need to recognize the member - -→ Easier to join, widely access, globally accept -> cryptocurrency - -## Hyperledger architecture - -### Identity - -The actors in a blockchain network(peers, orderers, client applications, administrators..) has a digital identity encapsulated in an X.509 digital certificate. - Properties of an actor's identity(organization, organizational unit, role..) wrap as Unique ID. - -### PKIs (public key infrastructure) - -Is a collection of internet technologies that provides secure communications in a network. - -Base on the properties of a peer, cryptography constructing a pair of public and private key(prevent reading messages by third parties), Certificate Authority(CA) is an entity that issues this digital certificates, allows relying parties to rely upon signatures or on assertions made about the private key that corresponds to the certified public key. So that, any message passing to a peer can only be read by this peer. - -- **Root CAs**: CAs as root -- **Intermediate CAs**: issued by root CA. -- **Fabric CA**: is a private root CA provider capable of managing digital identities if Fabric participants. -- **Certificate Revocation List(CRL)**: a list of references to certificates that a CA knows to be revoked. - -![](assets/fabric-hyperledger-architecture-explanation_acecde099998e363519533076028fb4e_md5.webp) - -### MSP (Membership Service Provider) - -Identifies which Root CAs and Intermediate CAs are trusted to define the members of a trust domain. Also identify specific roles an actor might play either within the scope of the organization the MSP represents, defining access privileges in the context of a network and channel. - -- **Organizational unit**: is a managed group of members(multinational corporation, flower shop) under a single MSP. -- **Local MSPs**: hold Root CAs of an Organization, authenticate at organization level(communications between peer, node). Only one local MSP per node or peer. -- **Channel MSPs**: hold Root CAs of connected Organization through channel to identify each other, there will be a local copy of channel MSP in each node or peer. - -**MSP level** - -- Network MSP: defines who are the members in the network by defining the MSPs of the participants organization. -- Channel MSP: provides private communications between a particular set of organizations. -- Peer MSP: is a local MSP provides private communications between peer belong to an Organization. -- Orderer MSP: is a local MSP, function like Peer MSP, only apply for node. - -![](assets/fabric-hyperledger-architecture-explanation_a9852ce4f9889dd96d9efe61fd1cdc0c_md5.webp) - -## Transaction flow from application - -- Applications generate a transaction proposal -- Send it to required peers ( indicate by transaction it own ) -- These peers the n become endorsing peers, then independently executes a chaincode -> proposal responses(difference peers can return different, inconsistent transaction responses, application is free to discard inconsistent transaction responses) - -![](assets/fabric-hyperledger-architecture-explanation_871c33102b9552789598d25986ccd406_md5.webp) - -- Orderer receives transactions containing endorsed transaction proposal responses from many applications -- Orders each transaction relative to other transactions -- packages batches of transactions into blocks ready for distribution back to all peers connected to the orderer ( stop signal of packaging phase: block of the desired size or after a maximum elapsed time) -- Strict order: transactions can be packaged in any order into a block, and it’s this sequence that becomes the order of execution -- No ledger fork: Once transactions are captured in a block, history cannot be rewritten for that transaction at a future point in time. - -![](assets/fabric-hyperledger-architecture-explanation_4717184a9972241d126ccab41d22390e_md5.webp) - -- Each transaction after order sending to peers within a block is validated by each peer. -- Failed transactions are retained for audit, but are not applied to the ledger -- Every time a block is committed to a peer's ledger, that peer generates an appropriate event(include full block content) - -![](assets/fabric-hyperledger-architecture-explanation_2e60ff9cb71f5ecb312418e4228c7964_md5.webp) diff --git a/topics/blockchain/general.md b/topics/blockchain/general.md deleted file mode 100644 index f694c83ef..000000000 --- a/topics/blockchain/general.md +++ /dev/null @@ -1,632 +0,0 @@ ---- -title: General -description: Learn best practices for Solidity smart contracts including abstract contracts vs interfaces, proper use of assert/require/revert, inheritance, event monitoring, fallback functions, security tips, and visibility modifiers. -date: null -redirect: - - /VQSEAA ---- - -### 1. Abstract-vs-interfaces - -Be aware of the tradeoffs between abstract contracts and interfaces. -Both interfaces and abstract contracts provide one with a customizable and re-usable approach for smart contracts. Interfaces, which were introduced in Solidity 0.4.11, are similar to abstract contracts but cannot have any functions implemented. Interfaces also have limitations such as not being able to access storage or inherit from other interfaces which generally makes abstract contracts more practical. Although, interfaces are certainly useful for designing contracts prior to implementation. Additionally, it is important to keep in mind that if a contract inherits from an abstract contract it must implement all non-implemented functions via overriding or it will be abstract as well. - -### 2. assert require revert - -#### Enforce invariants with `assert()` - -An assert guard triggers when an assertion fails - such as an invariant property changing. For -example, the token to ether issuance ratio, in a token issuance contract, may be fixed. You can -verify that this is the case at all times with an `assert()`. Assert guards should often be -combined with other techniques, such as pausing the contract and allowing upgrades. (Otherwise, you -may end up stuck, with an assertion that is always failing.) - -Example: - -```sol -contract Token { - mapping(address => uint) public balanceOf; - uint public totalSupply; - - function deposit() public payable { - balanceOf[msg.sender] += msg.value; - totalSupply += msg.value; - assert(address(this).balance >= totalSupply); - } -} -``` - -Note that the assertion is _not_ a strict equality of the balance because the contract can be forcibly sent ether without going -through the `deposit()` function! - -#### Use `assert()`, `require()`, `revert()` properly - -!!! Info -The convenience functions **assert** and **require** can be used to check for conditions and throw an exception if the condition is not met. - -The **assert** function should only be used to test for internal errors, and to check invariants. - -The **require** function should be used to ensure valid conditions, such as inputs, or contract state variables are met, or to validate return values from calls to external contracts. \* - -Following this paradigm allows formal analysis tools to verify that the invalid opcode can never be -reached: meaning no invariants in the code are violated and that the code is formally verified. - -```sol -pragma solidity ^0.5.0; - -contract Sharer { - function sendHalf(address payable addr) public payable returns (uint balance) { - require(msg.value % 2 == 0, "Even value required."); //Require() can have an optional message string - uint balanceBeforeTransfer = address(this).balance; - (bool success, ) = addr.call.value(msg.value / 2)(""); - require(success); - // Since we reverted if the transfer failed, there should be - // no way for us to still have half of the money. - assert(address(this).balance == balanceBeforeTransfer - msg.value / 2); // used for internal error checking - return address(this).balance; - } -} -``` - -See [SWC-110](https://swcregistry.io/docs/SWC-110) & [SWC-123](https://swcregistry.io/docs/SWC-123) - -## 3.Complex inheritance - -When utilizing multiple inheritance in Solidity, it is important to understand how the compiler -composes the inheritance graph. - -```sol - -contract Final { - uint public a; - function Final(uint f) public { - a = f; - } -} - -contract B is Final { - int public fee; - - function B(uint f) Final(f) public { - } - function setFee() public { - fee = 3; - } -} - -contract C is Final { - int public fee; - - function C(uint f) Final(f) public { - } - function setFee() public { - fee = 5; - } -} - -contract A is B, C { - function A() public B(3) C(5) { - setFee(); - } -} -``` - -When a contract is deployed, the compiler will _linearize_ the inheritance from right to left -(after the keyword _is_ the parents are listed from the most base-like to the most derived). Here -is contract A's linearization: - -**Final \<- B \<- C \<- A** - -The consequence of the linearization will yield a `fee` value of 5, since C is the most derived -contract. This may seem obvious, but imagine scenarios where C is able to shadow crucial functions, -reorder boolean clauses, and cause the developer to write exploitable contracts. Static analysis -currently does not raise issue with overshadowed functions, so it must be manually inspected. - -For more on security and inheritance, check out this -[article](https://pdaian.com/blog/solidity-anti-patterns-fun-with-inheritance-dag-abuse/) - -To help contribute, Solidity's Github has a -[project](https://github.com/ethereum/solidity/projects/9#card-8027020) with all -inheritance-related issues. - -See [SWC-125](https://swcregistry.io/docs/SWC-125) - -## 4. Event monitoring - -It can be useful to have a way to monitor the contract's activity after it was deployed. One way to -accomplish this is to look at all transactions of the contract, however that may be insufficient, -as message calls between contracts are not recorded in the blockchain. Moreover, it shows only the -input parameters, not the actual changes being made to the state. Also, events could be used to -trigger functions in the user interface. - -```sol -contract Charity { - mapping(address => uint) balances; - - function donate() payable public { - balances[msg.sender] += msg.value; - } -} - -contract Game { - function buyCoins() payable public { - // 5% goes to charity - charity.donate.value(msg.value / 20)(); - } -} -``` - -Here, `Game` contract will make an internal call to `Charity.donate()`. This transaction won't -appear in the external transaction list of `Charity`, but only visible in the internal -transactions. - -An event is a convenient way to log something that happened in the contract. Events that were -emitted stay in the blockchain along with the other contract data and they are available for future -audit. Here is an improvement to the example above, using events to provide a history of the -Charity's donations. - -```sol -contract Charity { - // define event - event LogDonate(uint _amount); - - mapping(address => uint) balances; - - function donate() payable public { - balances[msg.sender] += msg.value; - // emit event - emit LogDonate(msg.value); - } -} - -contract Game { - function buyCoins() payable public { - // 5% goes to charity - charity.donate.value(msg.value / 20)(); - } -} - -``` - -Here, all transactions that go through the `Charity` contract, either directly or not, will show up -in the event list of that contract along with the amount of donated money. - ---- - -!!! Note "Prefer newer Solidity constructs" -Prefer constructs/aliases such as `selfdestruct` (over `suicide`) and `keccak256` (over `sha3`). Patterns like `require(msg.sender.send(1 ether))` can also be simplified to using `transfer()`, as in `msg.sender.transfer(1 ether)`. Check out [Solidity Change log](https://github.com/ethereum/solidity/blob/develop/Changelog.md) for more similar changes. - -## 5. Extcodesize checks - -Avoid using `extcodesize` to check for Externally Owned Accounts. - -The following modifier (or a similar check) is often used to verify whether a call was made from an -externally owned account (EOA) or a contract account: - -```sol -// bad -modifier isNotContract(address _a) { - uint size; - assembly { - size := extcodesize(_a) - } - require(size == 0); - _; -} -``` - -The idea is straightforward: if an address contains code, it's not an EOA but a contract account. -However, **a contract does not have source code available during construction**. This means that -while the constructor is running, it can make calls to other contracts, but `extcodesize` for its -address returns zero. Below is a minimal example that shows how this check can be circumvented: - -```sol -contract OnlyForEOA { - uint public flag; - - // bad - modifier isNotContract(address _a){ - uint len; - assembly { len := extcodesize(_a) } - require(len == 0); - _; - } - - function setFlag(uint i) public isNotContract(msg.sender){ - flag = i; - } -} - -contract FakeEOA { - constructor(address _a) public { - OnlyForEOA c = OnlyForEOA(_a); - c.setFlag(1); - } -} -``` - -Because contract addresses can be pre-computed, this check could also fail if it checks an address -which is empty at block `n`, but which has a contract deployed to it at some block greater than -`n`. - -!!! Warning "This issue is nuanced." -If your goal is to prevent other contracts from being able to call your contract, the `extcodesize` check is probably sufficient. An alternative approach is to check the value of `(tx.origin == msg.sender)`, though this also has drawbacks. - - There may be other situations in which the `extcodesize` check serves your purpose. Describing all of them here is out of scope. Understand the underlying behaviors of the EVM and use your Judgement. - -## 6. Fallback functions - -#### Keep fallback functions simple - -[Fallback functions](http://solidity.readthedocs.io/en/latest/contracts.html#fallback-function) are -called when a contract is sent a message with no arguments (or when no function matches), and only -has access to 2,300 gas when called from a `.send()` or `.transfer()`. If you wish to be able to -receive Ether from a `.send()` or `.transfer()`, the most you can do in a fallback function is log -an event. Use a proper function if a computation of more gas is required. - -```sol -// bad -function() payable { balances[msg.sender] += msg.value; } - -// good -function deposit() payable external { balances[msg.sender] += msg.value; } - -function() payable { require(msg.data.length == 0); emit LogDepositReceived(msg.sender); } -``` - -#### Check data length in fallback functions - -Since the -[fallback functions](http://solidity.readthedocs.io/en/latest/contracts.html#fallback-function) is -not only called for plain ether transfers (without data) but also when no other function matches, -you should check that the data is empty if the fallback function is intended to be used only for -the purpose of logging received Ether. Otherwise, callers will not notice if your contract is used -incorrectly and functions that do not exist are called. - -```sol -// bad -function() payable { emit LogDepositReceived(msg.sender); } - -// good -function() payable { require(msg.data.length == 0); emit LogDepositReceived(msg.sender); } -``` - -## 7. integer division - -All integer division rounds down to the nearest integer. If you need more precision, consider using -a multiplier, or store both the numerator and denominator. - -(In the future, Solidity will have a -[fixed-point](https://solidity.readthedocs.io/en/develop/types.html#fixed-point-numbers) type, -which will make this easier.) - -```sol -// bad -uint x = 5 / 2; // Result is 2, all integer division rounds DOWN to the nearest integer -``` - -Using a multiplier prevents rounding down, this multiplier needs to be accounted for when working -with x in the future: - -```sol -// good -uint multiplier = 10; -uint x = (5 * multiplier) / 2; -``` - -Storing the numerator and denominator means you can calculate the result of `numerator/denominator` -off-chain: - -```sol -// good -uint numerator = 5; -uint denominator = 2; -``` - -## 8. interface types - -When a function takes a contract address as an argument, it is better to pass an interface or -contract type rather than a raw `address`. If the function is called elsewhere within the source -code, the compiler will provide additional type safety guarantees. - -Here we see two alternatives: - -```sol -contract Validator { - function validate(uint) external returns(bool); -} - -contract TypeSafeAuction { - // good - function validateBet(Validator _validator, uint _value) internal returns(bool) { - bool valid = _validator.validate(_value); - return valid; - } -} - -contract TypeUnsafeAuction { - // bad - function validateBet(address _addr, uint _value) internal returns(bool) { - Validator validator = Validator(_addr); - bool valid = validator.validate(_value); - return valid; - } -} -``` - -The benefits of using the `TypeSafeAuction` contract above can then be seen from the following -example. If `validateBet()` is called with an `address` argument, or a contract type other than -`Validator`, the compiler will throw this error: - -```sol -contract NonValidator{} - -contract Auction is TypeSafeAuction { - NonValidator nonValidator; - - function bet(uint _value) { - bool valid = validateBet(nonValidator, _value); // TypeError: Invalid type for argument in function call. - // Invalid implicit conversion from contract NonValidator - // to contract Validator requested. - } -} -``` - -## 9. Locking pragmas - -Contracts should be deployed with the same compiler version and flags that they have been tested -the most with. Locking the pragma helps ensure that contracts do not accidentally get deployed -using, for example, the latest compiler which may have higher risks of undiscovered bugs. Contracts -may also be deployed by others and the pragma indicates the compiler version intended by the -original authors. - -```sol -// bad -pragma solidity ^0.4.4; - -// good -pragma solidity 0.4.4; -``` - -Note: a floating pragma version (ie. `^0.4.25`) will compile fine with `0.4.26-nightly.2018.9.25`, -however nightly builds should never be used to compile code for production. - -!!! Warning -Pragma statements can be allowed to float when a contract is intended for consumption -by other developers, as in the case with contracts in a library or EthPM package. Otherwise, the -developer would need to manually update the pragma in order to compile locally. - - See [SWC-103](https://swcregistry.io/docs/SWC-103) - -## 10. Modifier - -The code inside a modifier is usually executed before the function body, so any state changes or -external calls will violate the -[Checks-Effects-Interactions](https://solidity.readthedocs.io/en/develop/security-considerations.html#use-the-checks-effects-interactions-pattern) -pattern. Moreover, these statements may also remain unnoticed by the developer, as the code for -modifier may be far from the function declaration. For example, an external call in modifier can -lead to the reentrancy attack: - -```sol -contract Registry { - address owner; - - function isVoter(address _addr) external returns(bool) { - // Code - } -} - -contract Election { - Registry registry; - - modifier isEligible(address _addr) { - require(registry.isVoter(_addr)); - _; - } - - function vote() isEligible(msg.sender) public { - // Code - } -} -``` - -In this case, the `Registry` contract can make a reentrancy attack by calling `Election.vote()` -inside `isVoter()`. - -!!! Note -Use [modifiers](https://solidity.readthedocs.io/en/develop/contracts.html#function-modifiers) to -replace duplicate condition checks in multiple functions, such as `isOwner()`, otherwise use -`require` or `revert` inside the function. This makes your smart contract code more readable and -easier to audit. - -## 11. payability - -Starting from Solidity `0.4.0`, every function that is receiving ether must use `payable` modifier, -otherwise if the transaction has `msg.value > 0` will revert -(except when forced). - -!!! Note -Something that might not be obvious: The `payable` modifier only applies to calls from _external_ contracts. If I call a non-payable function in the payable function in the same contract, the non-payable function won't fail, though `msg.value` is still set. - -## 12. Shadowing - -It is currently possible to [shadow](https://en.wikipedia.org/wiki/Variable_shadowing) built-in -globals in Solidity. This allows contracts to override the functionality of built-ins such as `msg` -and `revert()`. Although this [is intended](https://github.com/ethereum/solidity/issues/1249), it -can mislead users of a contract as to the contract's true behavior. - -```sol -contract PretendingToRevert { - function revert() internal constant {} -} - -contract ExampleContract is PretendingToRevert { - function somethingBad() public { - revert(); - } -} -``` - -Contract users (and auditors) should be aware of the full smart contract source code of any -application they intend to use. - -## 13. timestamp-dependence - -There are three main considerations when using a timestamp to execute a critical function in a -contract, especially when actions involve fund transfer. - -#### Timestamp manipulation - -Be aware that the timestamp of the block can be manipulated by a miner. Consider this -[contract](https://etherscan.io/address/0xcac337492149bdb66b088bf5914bedfbf78ccc18#code): - -```sol - -uint256 constant private salt = block.timestamp; - -function random(uint Max) constant private returns (uint256 result){ - //get the best seed for randomness - uint256 x = salt * 100/Max; - uint256 y = salt * block.number/(salt % 5) ; - uint256 seed = block.number/3 + (salt % 300) + Last_Payout + y; - uint256 h = uint256(block.blockhash(seed)); - - return uint256((h / x)) % Max + 1; //random number between 1 and Max -} -``` - -When the contract uses the timestamp to seed a random number, the miner can actually post a -timestamp within 15 seconds of the block being validated, effectively allowing the miner to -precompute an option more favorable to their chances in the lottery. Timestamps are not random and -should not be used in that context. - -#### The 15-second rule - -The [Yellow Paper](https://ethereum.github.io/yellowpaper/paper.pdf) (Ethereum's reference -specification) does not specify a constraint on how much blocks can drift in time, but -[it does specify](https://ethereum.stackexchange.com/a/5926/46821) that each timestamp should be -bigger than the timestamp of its parent. Popular Ethereum protocol implementations -[Geth](https://github.com/ethereum/go-ethereum/blob/4e474c74dc2ac1d26b339c32064d0bac98775e77/consensus/ethash/consensus.go#L45) -and -[Parity](https://github.com/paritytech/parity-ethereum/blob/73db5dda8c0109bb6bc1392624875078f973be14/ethcore/src/verification/verification.rs#L296-L307) -both reject blocks with timestamp more than 15 seconds in future. Therefore, a good rule of thumb -in evaluating timestamp usage is: - -!!! Note -If the scale of your time-dependent event can vary by 15 seconds and maintain integrity, -it is safe to use a `block.timestamp`. - -#### Avoid using `block.number` as a timestamp - -It is possible to estimate a time delta using the `block.number` property and -[average block time](https://etherscan.io/chart/blocktime), however this is not future proof as -block times may change (such as -[fork reorganisations](https://blog.ethereum.org/2015/08/08/chain-reorganisation-depth-expectations/) -and the [difficulty bomb](https://github.com/ethereum/EIPs/issues/649)). In a sale spanning days, -the 15-second rule allows one to achieve a more reliable estimate of time. - -See [SWC-116](https://swcregistry.io/docs/SWC-116) - -## 14. tx-origin - -Never use `tx.origin` for authorization, another contract can have a method which will call your -contract (where the user has some funds for instance) and your contract will authorize that -transaction as your address is in `tx.origin`. - -```sol -contract MyContract { - - address owner; - - function MyContract() public { - owner = msg.sender; - } - - function sendTo(address receiver, uint amount) public { - require(tx.origin == owner); - (bool success, ) = receiver.call.value(amount)(""); - require(success); - } - -} - -contract AttackingContract { - - MyContract myContract; - address attacker; - - function AttackingContract(address myContractAddress) public { - myContract = MyContract(myContractAddress); - attacker = msg.sender; - } - - function() public { - myContract.sendTo(attacker, msg.sender.balance); - } - -} -``` - -You should use `msg.sender` for authorization (if another contract calls your contract `msg.sender` -will be the address of the contract and not the address of the user who called the contract). - -You can read more about it here: -[Solidity docs](https://solidity.readthedocs.io/en/develop/security-considerations.html#tx-origin) - -!!! Warning -Besides the issue with authorization, there is a chance that `tx.origin` will be -removed from the Ethereum protocol in the future, so code that uses `tx.origin` won't be compatible -with future releases -[Vitalik: 'Do NOT assume that tx.origin will continue to be usable or meaningful.'](https://ethereum.stackexchange.com/questions/196/how-do-i-make-my-dapp-serenity-proof/200#200) - - It's also worth mentioning that by using `tx.origin` you're limiting interoperability between - contracts because the contract that uses tx.origin cannot be used by another contract as a contract - can't be the `tx.origin`. - - See [SWC-115](https://swcregistry.io/docs/SWC-115) - -## 15. Visibility - -Explicitly label the visibility of functions and state variables. Functions can be specified as -being `external`, `public`, `internal` or `private`. Please understand the differences between -them, for example, `external` may be sufficient instead of `public`. For state variables, -`external` is not possible. Labeling the visibility explicitly will make it easier to catch -incorrect assumptions about who can call the function or access the variable. - -- `External` functions are part of the contract interface. An external function `f` cannot be - called internally (i.e. `f()` does not work, but `this.f()` works). External functions are - sometimes more efficient when they receive large arrays of data. -- `Public` functions are part of the contract interface and can be either called internally or via - messages. For public state variables, an automatic getter function (see below) is generated. -- `Internal` functions and state variables can only be accessed internally, without using `this`. -- `Private` functions and state variables are only visible for the contract they are defined in and - not in derived contracts. **Note**: Everything that is inside a contract is visible to all - observers external to the blockchain, even `Private` - variables.\* - -```sol -// bad -uint x; // the default is internal for state variables, but it should be made explicit -function buy() { // the default is public - // public code -} - -// good -uint private y; -function buy() external { - // only callable externally or using this.buy() -} - -function utility() public { - // callable externally, as well as internally: changing this code requires thinking about both cases. -} - -function internalAction() internal { - // internal code -} -``` - -### See - -- [SWC-100](https://swcregistry.io/docs/SWC-100) and -- [SWC-108](https://swcregistry.io/docs/SWC-108) diff --git a/topics/blockchain/layer-2.md b/topics/blockchain/layer-2.md deleted file mode 100644 index c9e811b41..000000000 --- a/topics/blockchain/layer-2.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: "Layer 2: Scaling solutions for Ethereum" -description: "Explore Layer 2 solutions for Ethereum scaling, including rollups and their benefits. Learn how Layer 2 extends Ethereum's capabilities, reduces gas fees, and maintains security while improving transaction throughput and efficiency." -date: 2022-12-06 -authors: - - baenv -github_id: vhbien000 -tags: - - network - - blockchain - - ethereum -redirect: - - /rIxY9A ---- - -## What? - -- Ethereum scaling solutions -- Separate blockchain - - Extends Ethereum - - Inherits the security guarantees of Ethereum -- All user transactions on the Layer 2 can ultimately settle back to Layer 1 -- Ethereum also functions as a data availability layer for Layer 2s - - Layer 2 will post their transactions data onto Ethereum - - Rely on Ethereum for data availability - - Used to get the state of Layer 2 - - Dispute transaction of Layer 2 - -## Why? - -Blockchain has 3 desirable properties - -- Decentralized -- Secure -- Scalable - -"can only achieve 2 out of 3" - Blockchain trilemma - -High demand -> Need to scale without sacrificing decentralization and security => Need Layer 2 to scale Blockchain that takes advantage of robust decentralized security of Layer 1 - -## How? - -- Communicate with Layer 1 by submitting bundles of transactions -- Layer 1 handles security, data availability, and decentralization -- Layer 2 handles scaling by computing and sending finalized proofs to Layer 1 -> Remove transaction loading. - -### Rollup - -- Preferred layer 2 scaling solution in Ethereum -- Reduce gas fees by up to 100x compared to Layer 1 -- Rollup bundle ("roll up") hundreds of transactions into a Layer 1 transaction => Fee will be dived/distributed to all users (owners of these hundreds of transactions) -> Cheaper - - For example: - - 1 Layer 1 transaction is paid for 1eth as fees - - 100 Layer 2 transactions rolled up in 1 Layer 1 transaction are also paid for 1eth. So 1 Layer 2 transaction is just only needed 0.01 eth to execute. -- Rollup is executed outside Layer 1 (in Layer 2), but finalized result (proof) is submitted to Layer 1 => and can be secured by Layer 1 security mechanisms. -- Have 2 approaches (different on posting transaction data to L1): - - Optimistic - - ZK Rollups aka Zero-knowledge Rollups - -## Example - -- Arbitrum One -- Optimism -- Boba Network - -## References - -- https://ethereum.org/en/layer-2 diff --git a/topics/blockchain/liquidity-pool.md b/topics/blockchain/liquidity-pool.md deleted file mode 100644 index 77746768f..000000000 --- a/topics/blockchain/liquidity-pool.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: Liquidity pool -description: A comprehensive guide to understanding liquidity pools in blockchain, covering their definition, functionality, the role of automated market makers (AMM), arbitrage, pool depth, slippage, how to provide liquidity, and the concept of impermanent loss. This article provides insights for both beginners and experienced users in decentralized finance (DeFi) -date: 2022-03-24 -authors: - - leduyhien152 -github_id: leduyhien152 -tags: - - blockchain - - liquidity -redirect: - - /dPGSLA ---- - -## 1. What is a Liquidity Pool? - -Basically, a liquidity pool is a pool of tokens that are locked in a smart contract. Liquidity refers to the ease with which a token can be swapped with another. Anybody can provide liquidity into this single giant pool and earn a share of the trading fees based on their stake in it. - -The process could be illustrated by the picture below: - -![](assets/liquidity-pool_pasted-image-20220322220453.webp) - -## 2. How do Liquidity Pools work? - -Liquidity pools form the backbone of DEX by applying the automated market maker (AMM) system. Here’s the main formula that mathematically determines what the market price of the token in the pool should be: - -$$ -x * y = k -$$ - -Where x and y represent the respective token balance of a pairing and **k is a constant that will never change**. - -Let's use the ETH-DAI pair as an example, with 10 ETH and 1,000 DAI in the liquidity pool. What happens when someone wants to buy 1 ETH from this pool? How much does he need to pay? - -The k constant is 10,000 since there are 10 ETH and 1,000 DAI. - -$$ -10 \text{ ETH} * 1,000 \text{ DAI} = 10,000 -$$ - -If the buyer withdraws 1 ETH, he has to deposit some DAI into the pool so that k remains constant. - -$$ -(10 - 1) \text{ ETH} * (1,000 - y) = 10,000 -$$ - -$$ -1,000 - y =\frac{10000}{10 - 1} -$$ - -$$ - y = 111.11 -$$ - -And because we have no limit orders in AMM, the smart contract would automatically compute y to determine the price to pay and that is approximately 111.11 DAI. - -Now the liquidity pool would have 9 ETH and 1,111.11 DAI after someone buys 1 ETH. - -### 2.1. Roles of arbitragers in AMM - -Arbitragers play an important role in AMM. They are used in order to take advantage of the price differences and drive the price back towards market equilibrium. - -If the price of ETH in the pool is higher than it is on Coinbase, arbitragers would sell ETH into this pool and make a profit on the price discrepancies. Thus, the price of ETH in the pool would always be incentivized towards the market price as closely as possible. - -### 2.2. Depth of pool and slippage - -The price difference between the pool and market is known as slippage. How big your slippage is depends on the size of your trade relative to the size or depth of the pool. - -The depth of the pool is measured by that k constant. The bigger your k, the deeper the pool and the less likely a slippage is going to occur. - -In the earlier example, buying 1 ETH from a pool that only has 10 ETH makes up 10% of the pool size. Hence there is such a big difference in price. ETH price costs $100 but you are buying it from the pool at $111.11. That’s about an 11% price slippage. - -In reality, the pool will be much deeper and bigger as there will be hundreds and thousands of liquidity providers from all around the world. - -Suppose we use a pool that has 100 ETH and 10,000 DAI and someone wants to buy 1 ETH from this pool, how much would it cost? Plugging in the same equation would give you $101, a 1% price slippage. - -## 3. How to provide Liquidity into a Pool? - -Anyone can provide liquidity and become a Liquidity Provider (LP). When supplying a pair of tokens into the pool, the ratio price of both tokens must be 50-50. So if you want to provide $5,000 of ETH-DAI pair, you will need $2,500 DAI and $2,500 worth of ETH. - -Every liquidity provider has to follow this standard so that the liquidity pool would always maintain a 50-50 mix of token A and token B. - -When you provide liquidity into a pool, you typically receive an LP token in exchange. This LP token represents your share in the liquidity pool. Every time when a trade is made on the liquidity pool, users have to pay a fee. These fees are then aggregated and re-distributed back to all liquidity providers on a pro-rata basis based on the amount of LP tokens you hold. - -However, you may not get back the exact amount of tokens you deposited initially. That is to say, if you started out with some ETH-DAI tokens, you would get back more ETH and less DAI, or more DAI and less ETH depending on the markets. - -In a bull market, more people would want ETH as prices are rising. Hence the supply of ETH in the pool would drop while DAI would increase since more people are exchanging their dollars for ETH. When you withdraw out your LP, you would end up with less ETH than you started out with and more DAI. The reverse holds true in a bear market. - -## 4. What is Impermanent Loss? - -Impermanent loss refers to the situation where you could have made more if you have done nothing and hold on for dear life rather than providing LP. - -Suppose the price of ETH in our LP is $100. What if the price of ETH on Coinbase rises to $120 in the market? Arbitragers will come in and buy ETH from the pool and sell it on Coinbase to profit from that difference. - -Let’s use a pool that has 100 ETH and 10,000 DAI. The relation between x, y, k, and ETH price could be shown by: - -$$ -x * y = k -$$ - -$$ -x = \frac{k}{\text{ETH price}} -$$ - -We could easily calculate x and y by k and ETH price: - -$$ -x = \sqrt{\frac{k}{\text{ETH price}}} -$$ - -$$ -y = \sqrt{k * \text{ETH price}} -$$ - -Assume someone supplies 1 ETH and 100 DAI into the pool. How much ETH and DAI he could get back if the ETH price pumps to $120? - -$$ -k = 100 * 10,000 = 1,000,000 -$$ - -$$ -x = \sqrt{\frac{1,000,000}{120}} = 91.29 -$$ - -$$ -y = \sqrt{1,000,000 * 120} = 10,954.45 -$$ - -Since his share in the pool is 1%, the LP gets back 0.9129 ETH and 109.5445 DAI if he wants to withdraw his stake in the pool. The total value of his stake would be 0.9129 ETH \* $120 + $109.54, which totals up to be $219.09. - -If he did not provide his liquidity into the pool and held on for dear life instead, his initial asset would be worth 1 ETH \* $120 + $100 = $220, which also means he would have made an extra $0.91. - -That is what we call impermanent loss. It is impermanent because it only becomes permanent when you withdraw out your LP. - ---- - -#### Reference - -- https://www.jumpstartmag.com/how-do-crypto-liquidity-pools-work/ -- https://www.theancientbabylonians.com/what-is-liquidity-pool-lp-in-defi/#:~:text=To%20sum%20up%20what%20liquidity,%3A%20x%20*%20y%20%3D%20k diff --git a/topics/blockchain/multisign-wallet.md b/topics/blockchain/multisign-wallet.md deleted file mode 100644 index 763126ab5..000000000 --- a/topics/blockchain/multisign-wallet.md +++ /dev/null @@ -1,274 +0,0 @@ ---- -title: Multisign wallet -description: A multisign wallet is a type of digital wallet that requires multiple signatures to authorize transactions. This article provides an overview of multisign wallets, how they work, and their benefits. -date: 2022-08-10 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - multisign-wallet -redirect: - - /oPw4zQ ---- - -![](assets/multisign-wallet-hero-image.webp) - -## Intro multisig wallet - -Usually, blockchain wallets are generally generated by a unique private key. All assets or transactions are controlled and signed by that private key. That key holder can do anything with that wallet. This is great for individuals, as it ensures privacy and security, as only the owner of the private key has full rights to those assets. - -But with an organization, such as assets after IDO, private sale, or common company assets, it is different, if this asset is controlled by a single person, there will be a lot of risk. can happen like: - -- This person took all the assets and disappeared. -- Due to some reason, this person loses the private key, then recovering the private key is not possible, which means that the entire property will also be inaccessible. -- If this person unfortunately dies, no one can inherit or use the property anymore because there is no private key. - -To solve the above problems, **Multisig Wallet** was born to minimize those risks. - -## What is multisig wallet? - -In essence _Multisig Wallet_ is a **smart contract** on the blockchain that allows certain logic to be processed when there are **enough** required signatures. - -> _Multisig_ is short for _Multi Signature_. - -A _Multisig Wallet_ has the following properties: - -- Parties agree on an action -- The rules in the smart contract have been built -- Smart contract can receive cryptocurrency (e.g. Ether) -- Smart contract can receive request, can be able to process that request based on consensus signing -- According to the multisig address configuration, it may require a different key combination: 2-of-3 is the most common key, where only 2 signatures are enough to access the funds of an address 3 signature. However, there are many other variations, such as 2 of 2, 3 of 3, 3 of 4, etc. - -## How does it work? - -As a simple analogy, we can picture a safe with two locks and two keys. One key is held by Alice and the other is held by Bob. The only way they can open the box is to provide both keys at the same time, so one cannot open the box without the other's consent. - -Basically, funds stored on a multi-signature address can only be accessed using 2 or more signatures. Thus, using a multisig wallet allows users to create an extra layer of security for their funds. But before going any further, it is important to understand the basics of standard Bitcoin addresses, which are based on a single key rather than multiple (single key addresses). - -## Pros and cons of multisig wallet - -Everything has two sides, security and convenience are always two opposite sides of each other, so is Multisig Wallet. - -### Advantages - -- Higher level of safety for any web3 user. -- Multisig Wallet enhances the security of assets: For example, we can set up a multisig wallet for 3 accounts at 3 devices: phone, tablet, laptop. Each transaction can only be done when there is confirmation from 2 of those 3 devices. At this time, if we assume that we lose our phone, we can still confirm the transaction using tablets and laptops, while the thief cannot confirm the transaction with just one device. -- Multisig Wallet dispute resolution: An example is A and B buying and selling assets, they decide to use a 2-of-3 multisig wallet with the participation of 3 parties A, B, and ruling party C. In case A and B have signed _agree_ or _cancelled_, the participation of C will not be needed. Otherwise if only one sign _co-sign_, and the other person signs _cancellation_, then C's signature will decide whether the transaction will be _agreeed_ or _cancelled_ by a majority of 2 out of 3 decisions. -- Decision making: A board of directors might use a multisig wallet to control access to a company's funds. For example, by setting up a 4-of-6 wallet where each board member holds one key, no individual board member is able to misuse the funds. Therefore, only decisions that are agreed upon by the majority can be executed. - -### Defect - -- In the above example, suppose we need 3 devices to confirm the transaction, and we lose our phone? neither me nor the thief can confirm any more transactions? That's the downside of Multisig Wallet: the rules are too complicated to set up effectively for each specific problem. - -> In this case, 2FA can be an effective solution to save the backup code for the account on the device, if we lose the device, we can still get the backup code back, which means getting the account back. . - -- Both blockchain and multisig are still new technologies, the security audit is still limited. There is no 100% guarantee that what works today won't be hacked tomorrow. For example, the $300 million hack of Parity Wallet. -- More gas fee when you have to submit your signature to verify a request and deplay to making final transaction. -- Still limited action by smart contract operations. - -## Build minimum multisig wallet - -[Mutisig smart contract](https://solidity-by-example.org/app/multi-sig-wallet/): - -```js -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.13; - -contract MultiSigWallet { - event Deposit(address indexed sender, uint amount, uint balance); - event SubmitTransaction( - address indexed owner, - uint indexed txIndex, - address indexed to, - uint value, - bytes data - ); - event ConfirmTransaction(address indexed owner, uint indexed txIndex); - event RevokeConfirmation(address indexed owner, uint indexed txIndex); - event ExecuteTransaction(address indexed owner, uint indexed txIndex); - - address[] public owners; - mapping(address => bool) public isOwner; - uint public numConfirmationsRequired; - - struct Transaction { - address to; - uint value; - bytes data; - bool executed; - uint numConfirmations; - } - - // mapping from tx index => owner => bool - mapping(uint => mapping(address => bool)) public isConfirmed; - - Transaction[] public transactions; - - modifier onlyOwner() { - require(isOwner[msg.sender], "not owner"); - _; - } - - modifier txExists(uint _txIndex) { - require(_txIndex < transactions.length, "tx does not exist"); - _; - } - - modifier notExecuted(uint _txIndex) { - require(!transactions[_txIndex].executed, "tx already executed"); - _; - } - - modifier notConfirmed(uint _txIndex) { - require(!isConfirmed[_txIndex][msg.sender], "tx already confirmed"); - _; - } - - constructor(address[] memory _owners, uint _numConfirmationsRequired) { - require(_owners.length > 0, "owners required"); - require( - _numConfirmationsRequired > 0 && - _numConfirmationsRequired <= _owners.length, - "invalid number of required confirmations" - ); - - for (uint i = 0; i < _owners.length; i++) { - address owner = _owners[i]; - - require(owner != address(0), "invalid owner"); - require(!isOwner[owner], "owner not unique"); - - isOwner[owner] = true; - owners.push(owner); - } - - numConfirmationsRequired = _numConfirmationsRequired; - } - - receive() external payable { - emit Deposit(msg.sender, msg.value, address(this).balance); - } - - function submitTransaction( - address _to, - uint _value, - bytes memory _data - ) public onlyOwner { - uint txIndex = transactions.length; - - transactions.push( - Transaction({ - to: _to, - value: _value, - data: _data, - executed: false, - numConfirmations: 0 - }) - ); - - emit SubmitTransaction(msg.sender, txIndex, _to, _value, _data); - } - - function confirmTransaction(uint _txIndex) - public - onlyOwner - txExists(_txIndex) - notExecuted(_txIndex) - notConfirmed(_txIndex) - { - Transaction storage transaction = transactions[_txIndex]; - transaction.numConfirmations += 1; - isConfirmed[_txIndex][msg.sender] = true; - - emit ConfirmTransaction(msg.sender, _txIndex); - } - - function executeTransaction(uint _txIndex) - public - onlyOwner - txExists(_txIndex) - notExecuted(_txIndex) - { - Transaction storage transaction = transactions[_txIndex]; - - require( - transaction.numConfirmations >= numConfirmationsRequired, - "cannot execute tx" - ); - - transaction.executed = true; - - (bool success, ) = transaction.to.call{value: transaction.value}( - transaction.data - ); - require(success, "tx failed"); - - emit ExecuteTransaction(msg.sender, _txIndex); - } - - function revokeConfirmation(uint _txIndex) - public - onlyOwner - txExists(_txIndex) - notExecuted(_txIndex) - { - Transaction storage transaction = transactions[_txIndex]; - - require(isConfirmed[_txIndex][msg.sender], "tx not confirmed"); - - transaction.numConfirmations -= 1; - isConfirmed[_txIndex][msg.sender] = false; - - emit RevokeConfirmation(msg.sender, _txIndex); - } - - function getOwners() public view returns (address[] memory) { - return owners; - } - - function getTransactionCount() public view returns (uint) { - return transactions.length; - } - - function getTransaction(uint _txIndex) - public - view - returns ( - address to, - uint value, - bytes memory data, - bool executed, - uint numConfirmations - ) - { - Transaction storage transaction = transactions[_txIndex]; - - return ( - transaction.to, - transaction.value, - transaction.data, - transaction.executed, - transaction.numConfirmations - ); - } -} -``` - -### Deployed multisig wallets - -There is no universal standard for writing Multisig Wallet, but we can refer to the implementation from famous wallets being used in the world to be able to design or inherit our own implementation. - -- [ConsenSys' multisig wallet](https://github.com/ConsenSys/MultiSigWallet): This can be considered the simplest implementation of Multisig Wallet, the solidity version used is also 0.4.10 a long time ago. , but is extremely valuable, at the time of writing this wallet is holding 80,000 Ether, or about 17 million dollars. You can look up this wallet [here](https://etherscan.io/address/0x851b7f3ab81bd8df354f0d7640efcd7288553419). -- [Gnosis' multisig wallet](https://github.com/Gnosis/MultiSigWallet): is an upgraded version of Consensys Multisig Wallet, written according to Truffle project's structure, fully tested and regularly updated. At the time of writing, this github project is still being updated. -- [Gnosis' multisig wallet](https://github.com/safe-global/safe-contracts): is an upgraded version of Consensys Multisig Wallet, written according to the structure of the hardhat project. -- [BitGo's multisig wallet](https://github.com/BitGo/eth-multisig-v2): also a structured version of Truffle, fully tested and regularly updated. The difference here is that the contract has more complex logic, one of which is ERC20-Token Compatibility. And this wallet implements 2-of-3 signing method, which means that there are exactly 3 parties involved, and 2 signatures are needed to agree for a transaction to take place. -- [Ethereum dapp's multisig wallet](https://github.com/ethereum/dapp-bin/blob/master/wallet/wallet.sol): Compatible with Ethereum Wallet or Mist, we can deploy Multisig Wallet here, and easily call `send transaction` or `confirm` directly. However, there is a disadvantage that there is no document, we have to read the code for more details. -- [Parity's multisig wallet](https://parity.io/) (NOT RECOMMENDED): This is also a very famous Multisig Wallet before it was [hacked and lost $300 million](https://medium.com /chain-cloud-company-blog/parity-multisig-hack-again-b46771eaa838) on 11/06/2017. The reason is because the implementation is not good, and therefore it is not recommended anymore. - -## Referrence - -- https://www.curvegrid.com/docs/multi-signature-multisig-wallet-smart-contracts -- https://www.binance.vision/security/what-is-a-multisig-wallet -- https://medium.com/hellogold/ethereum-multi-signature-wallets-77ab926ab63b -- https://www.gemini.com/cryptopedia/what-is-a-multi-sig-wallet-crypto-multi-signature-wallet diff --git a/topics/blockchain/nft-fractionalization.md b/topics/blockchain/nft-fractionalization.md deleted file mode 100644 index 5be006b42..000000000 --- a/topics/blockchain/nft-fractionalization.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: NFT fractionalization -description: NFT fractionalization is a process of dividing a non-fungible token (NFT) into smaller, fractional tokens that represent partial ownership of the original NFT. This article provides an overview of fractional NFTs, how they work, and their benefits. -date: 2022-06-18 -authors: - - trankhacvy -github_id: trankhacvy -tags: - - blockchain - - nft -redirect: - - /vmLciA ---- - -## What is a fractional NFT? - -- A fractional NFT is simply a whole NFT that has been divided into smaller fractions, allowing different numbers of people to claim ownership of a piece of the same NFT. The NFT is fractionalized using a smart contract that generates a set of tokens linked to the indivisible original. -- These fractional tokens give each holder a percentage of ownership of an NFT, and can be traded or exchanged on secondary markets. - -## How does it work? - -To break this NFT into fractions, it must first be locked in a smart contract, which will split the ERC-721 token into multiple ERC-20 tokens as per the instructions the NFT owner has given. The owner specifies everything, from the number of ERC-20 tokens to be created, to their prices, to the metadata to be used, to any other property they deem important. Each fraction or ERC-20 token created represents partial ownership of the NFT. The fractions can then be put up for sale at a fixed price for a particular time or until they get sold out. - -![](assets/nft-fractionalization_y8tfcaq.webp) - -## Benefits of fractional NFT - -- **Price discovery**: the asset is extremely valuable and they want help finding price discovery, fractionalizing the item and selling 20% on the market can be a valuable tool to help understand how the market values the NFT. -- **More liquidity**: owners have significantly better exit liquidity than if they owned the NFT themselves. This can be achieved through on-chain exchanges such Uniswap. -- **Curator incentives**: an NFT owner who divides their asset into fractions receives a curator fee from their chosen NFT marketplace. Although the owner can set and update the amount of this fee, it is subject to a maximum price limit to prevent reckless pricing. - -## References - -- https://www.coindesk.com/learn/how-can-you-share-an-nft-fractional-nfts-explained/ -- https://learn.bybit.com/nft/what-are-fractional-nfts/ -- https://www.leewayhertz.com/fractional-nft/ -- https://medium.com/fractional-art/what-is-fractional-dd4f86e6458a diff --git a/topics/blockchain/plonky2.md b/topics/blockchain/plonky2.md deleted file mode 100644 index f8e489843..000000000 --- a/topics/blockchain/plonky2.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: Plonky2 -description: "Plonky2 is a non-Interactive Proof of Proof of Knowledge (NIPoPoK) chain that reduces blockchain size and speeds up transaction confirmations. This article explores Plonky2's technology, its applications in various blockchain projects, and compares it with other scaling solutions." -date: 2023-02-28 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - proof-of-knowledge -redirect: - - /WoNPHg ---- - -## Introduction - -![](assets/plonky2_plonky-2-website.webp) Plonky2 is a non-Interactive Proof of Proof of Knowledge (NIPoPoK) chain developed to reduce the size of the blockchain and speed up transaction confirmations. It is built on the foundation of ZK-SNARKs (Zero-Knowledge Succinct Non-Interactive Argument of Knowledge), an advanced cryptographic technology that enables affirmative proofs without revealing inside information. Plonky2 allows transactions to be confirmed on the fly using certificates similar to Merkle trees, where only a small portion of data is needed to confirm the entire Merkle tree. Plonky2 certificates are very small, only about 1/20th of similar attestations used in traditional blockchain systems, thus helping to reduce blockchain size and speed up transaction processing. Plonky2 is being used in several blockchain projects such as Ethereum and Polkadot to improve the performance and scalability of the system. - -## The target of Plonky2 - -Blockchain is a decentralized information storage and conversion technology that allows parties to exchange information and transact directly with each other without the involvement of an intermediary. Transactions are confirmed and written to a new block in the blockchain, forming a reliable and unmodifiable history of transactions. An important issue of blockchain is its size. Since each new block added to the blockchain must be stored and backed up across the entire network, the size of the blockchain increases over time. This leads to problems with transaction processing speed, scalability, and storage costs. - -Some blockchains like Bitcoin use Proof of Work (PoW) to verify new transactions and generate new blocks. However, the use of PoW requires huge computation and high energy usage, negatively affects the environment and is costly. So many new technologies have been developed to solve the blockchain size problem, speed up transaction processing and reduce storage costs. Plonky2 is one of those new technologies, with the ability to reduce blockchain size significantly using non-interactive attestation (NIPoPoK) and Merkle tree structure. - -Plonky2 is a new technology in the blockchain field, developed to reduce blockchain size and speed up transaction confirmation by using non-interactive attestation (NIPoPoK) and Merkle tree structure. NIPoPoK is a method of building attestations for blocks in the blockchain without downloading the entire blockchain, requiring only a small number of blocks (usually only about 1% of the total block) called "superblocks". With NIPoPoK, confirmation of new transactions becomes faster and requires less resources than confirmations on the entire blockchain. - -The Merkle tree is a layered data structure that allows for fast and efficient data integrity checking. Using the Merkle tree structure in Plonky2, the blocks of the blockchain can be grouped into smaller "superblocks", which reduces the blockchain size significantly. The use of Plonky2 in the blockchain will help speed up transaction confirmations and reduce blockchain size, while reducing storage costs and increasing the scalability of the system. Plonky2 has been used in many blockchain projects, such as Ethereum and Bitcoin Cash. - -## How Plonky2 work - -Non-Interactive attestation (NIPoPoK) is a method of attesting in the blockchain without interacting with other entities in the system. Instead of downloading the entire blockchain to confirm a block, NIPoPoK requires only a small number of blocks (usually only about 1% of the total block) called "superblocks" to verify the validity of a new block. The main difference between NIPoPoK and interactive attestation (PoK) is that PoK requires interaction between entities in the system for authentication. This means that entities must exchange information with each other to perform authentication. Meanwhile, NIPoPoK does not require interaction between entities, which minimizes communication and synchronization in the system. The other difference between NIPoPoK and PoK is that PoK typically uses more complex protocols for authentication, while NIPoPoK uses simpler maths to compute authentication. However, this does not mean that NIPoPoK is less secure, because NIPoPoK uses the latest techniques to ensure the safety and security of data. With the development of technology, NIPoPoK is being widely used in blockchain to minimize the interaction and synchronization in the system, while helping to speed up transaction confirmation and reduce blockchain size. - -Merkle tree is a binary tree data structure used to store data and perform data integrity validation efficiently. Merkle trees are widely used in blockchain systems to minimize data size and speed up transaction integrity validation. Merkle tree is created by breaking data into smaller elements, called leaf nodes, then creating pairs of these leaves and hashing them together to form new nodes, until there are only leaves left. a single node at the top of the tree, called the root node. When there is a change in the data, simply recalculate the hash of the affected nodes and the root node will change, helping to verify the integrity of the data quickly and efficiently. - -Plonky2 uses a Merkle tree to verify transaction integrity. Instead of validating the entire blockchain like other methods, Plonky2 simply uses a small number of blocks (superblocks) created by the Merkle tree to verify the validity of a new transaction. Since then, Plonky2 reduces the size of data to be processed and speeds up transaction validation significantly. Merkle tree is used to generate superblocks, where each superblock is created by taking root nodes of Merkle trees for a certain period of time. Superblocks are stored in a linked list, and Plonky2 uses this list to verify the integrity of the new transaction. If a transaction is invalid, Plonky2 will re-query the superblock to find the invalid transaction and thereby confirm the integrity of the system. - -## Application of Plonky2 - -### Blockchain projects using Plonky2 and how Plonky2 improves system performance and scalability: - -- Mina Protocol: This is currently the smallest blockchain with a size of only about 22KB, using Plonky2 to reduce the size of the blockchain and increase transaction processing speed. -- Coda Protocol: Similar to Mina Protocol, Coda Protocol uses Plonky2 to reduce the size of the blockchain and increase transaction processing speed. -- Manta Network: Manta Network is a decentralized finance (DeFi) platform on the Polkadot network, using Plonky2 to reduce the size of the blockchain and increase transaction processing speed. -- Ergo: Ergo is a new decentralized blockchain, using Plonky2 to reduce the size of the blockchain and increase transaction processing speed. - -### Example for using Plonky2 - -An example of the use of Plonky2 is in the blockchain system of rewards points for loyalty programs. This system uses Plonky2 to reduce the size of the blockchain and speed up transaction processing. - -In this system, each time a customer makes a purchase or uses the company's service, they will be allocated a number of reward points corresponding to the amount spent. These reward points are recorded on the blockchain and can be used in exchange for other products or services of the company. However, because the number of customers and the number of transactions in the system can be very large, recording every transaction on the blockchain can lead to the blockchain being too large and the transaction processing slow. - -To solve this problem, the system uses Plonky2 to reduce the size of the blockchain. Instead of recording every transaction on the blockchain, the system only stores the transaction summary (transaction summary) as a Merkle tree. Plonky2 is used to confirm these transaction summaries and prove that all the transactions that have been performed are valid. As a result, this reward points blockchain system can process thousands of transactions per second, and the blockchain size increases only slightly if the number of customers and transactions increases. - -## Challenge of Plonky2 - -### Safety and security issues when using Plonky2 - -![](assets/plonky2_tradeoffs.webp) - -During authentication, Plonky2 asks the nodes to specify a specific number of Merkle tree elements. However, if a node sends incorrect elements or changes the Merkle tree, it can cause validation errors and affect the integrity of the blockchain. Therefore, ensuring the correctness of the Merkle tree is very important. - -In addition, Plonky2 also depends on the security of the hash system and the elliptic curve. If one of these factors is broken, then the authentication will not be secure and lead to security problems. Another problem for Plonky2 is a preimage attack, where an attacker uses previously known information to find secret hashes. If attacked, Plonky2 will no longer guarantee the integrity of the data. Therefore, when using Plonky2, developers need to pay attention to security and safety issues to ensure the integrity and security of the blockchain system. - -### Challenges in implementing and using Plonky2 on different blockchain systems - -- Plonky2 Complexity: Plonky2 is a new technology and its complexity can make it difficult to deploy on different blockchain systems. -- Compatibility: Some blockchain systems have separate processes and features, so integrating Plonky2 into these systems can be difficult. -- Reliability and security: Deploying Plonky2 requires the reliability and security of the software, and a security incident can lead to the loss of assets and currency. -- Processing speed: Plonky2 can help speed up transaction processing, but implementation on different blockchain systems may require time and effort to optimize performance. -- Scalability: Plonky2 is a highly scalable technology, but deployment on different blockchain systems may require different configurations and management to achieve maximum scalability. . - -## Compare Plonky 2 with others - -- Lightning Network: Lightning Network is a technology developed on the Bitcoin platform to reduce the cost and time of transaction processing. Lightning Network uses an offline payment channel to minimize communication between network nodes and speed up transaction processing. However, Lightning Network does not solve the blockchain size problem like Plonky2. -- Sharding: Sharding is a method of distributing data across many different nodes to reduce the load on each node and increase transaction processing speed. However, Sharding also faces many challenges in terms of security and compatibility with other systems. -- Sidechain: Sidechain is a technology that allows the creation of sub-blockchains to process different transactions. These sub-blockchains are linked to the main blockchain and can offload the main blockchain. However, linking between the secondary blockchains and the main blockchain can cause some security issues. - -### Comparison between Plonky2 and other technologies in improving the performance and scalability of the blockchain system: - -- Plonky2 vs. SegWit: SegWit (Segregated Witness) is a solution used on the Bitcoin network to reduce transaction size and increase processing speed. However, SegWit cannot solve the problems related to network scalability. Meanwhile, Plonky2 uses a number of different techniques such as Merkle tree and non-interactive attestation to reduce blockchain size and speed up transaction processing, while improving the scalability of the system. -- Plonky2 vs. Lightning Network: Lightning Network is a technology used to reduce costs and speed up transaction processing on the Bitcoin network. However, Lightning Network cannot solve the problem of network scalability. Meanwhile, Plonky2 reduces the blockchain size and increases transaction processing speed without compromising the safety and security of the system, while improving the scalability of the network. -- Plonky2 vs. Sharding: Sharding is a solution used to improve the scalability of the blockchain system by dividing the blockchain into parts and processing them independently. However, Sharding has limitations on the safety and security of the system. Meanwhile, Plonky2 helps to reduce blockchain size and speed up transaction processing without affecting the safety and security of the system. -- Plonky2 vs. Plasma: Plasma is a solution used to improve Ethereum's scalability by layering and processing transactions on sidechains. - -## Reference - -- https://polygon.technology/blog/plonky2-a-deep-dive -- https://github.com/mir-protocol/plonky2 -- https://hackernoon.com/zero-knowledge-proof-algorithm-plonk-circuit-sin7y-tech-review-16 -- https://vitalik.ca/general/2019/09/22/plonk.html -- https://zkresear.ch/t/composition-of-proof-systems/43 diff --git a/topics/blockchain/polygon-zkevm-architecture.md b/topics/blockchain/polygon-zkevm-architecture.md deleted file mode 100644 index 45854cf71..000000000 --- a/topics/blockchain/polygon-zkevm-architecture.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: Polygon zkEVM architecture -description: This article provides an overview of the Polygon zkEVM architecture, including its main components, how it works, and its benefits. -date: 2023-01-03 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - evm - - polygon - - zk-rollup -redirect: - - /W0MdwQ ---- - -## Polygon zkEVM architecture - -The main purpose of this architecture is Efficiency, the first strategy is incentivize the most efficient aggregators to participate in the proof generation process. The second is move all computation off-chain but keep only the necessary data zk-proof on-chain. Make the bridge decentralize. Utilization of special cryptographic primitive within the zkProver in order to speed up computation and minimize proof size. - -![](assets/polygon-zkevm-architecture_polygon-zkevm.webp) - -### Main components: - -- **Proof of efficiency** is a consensus mechanism based on an automatically conducted decentralized auction, with participants bidding on a certain amount of tokens to be selected to create the next batch added support for permissionless participation of multiple coordinators to create batches in L2 -- **zkNode** is software to be run by any zkEVM node, it not required to install Synchronization and adjust the roles of the participants. They can join as a node to know the state of the network, or can participate in production processing in any role: Sequencer or Aggregator. - - Synchronizer - - Sequencer & Aggregators - - RPC - -![](assets/polygon-zkevm-architecture_fig3-zknode-arch-aa4d18996fba1849291ea18e3f11d955.webp) - -- **zkProver** is techinal to create validity proof using zero-knowledge. it consists of a main state machine executor ( a collection of secondary state machines), a STARK proof builder, and a SNARK-proof builder. All valid batch must satisfy specific polynomial constraint. - -![](assets/polygon-zkevm-architecture_polygon-zkprover.webp) - -- **LX-to-Ly bridge** is a smart contract can help users transfer assets between two layers. - -### How does it work? - -#### What is Proof of efficiency? - -PoE solve the problem relate to decentralized and permissionless validators in Layer 2. Using zk-STARK for proving purpose, this proof are very fast but they are very big size. So, using zk-SNARK to attest to the correctness of the zk-STARK proofs. This help in reducing the gas cost from 5M to 350k. - -**Sequencer** create a batch of Layer 2 transaction from users and so select and pre-process a new L2 batch in network by sending a L1 tx with the data of all selected Layer 2 TXs. The transaction in L2 will be in format on L1 transaction with information in the **CALLDATA**, it will be used as the data available for the L2 network and L2 node will be able to synchronize the state. The new state is settled (validity proof of new state is generated and mined in L1) these data availability on L1 transaction define the L2 TXs that will be executed in specific order. - -The batch is process when the sequencer to do base on the incentives they have: - -![](assets/polygon-zkevm-architecture_1b54ce784c821f34b8d5d7218850095a84c9e054.webp) - -**Aggregators** receives all transaction information form Sequencer and send it to prover to get proof and send proof to smart contract to check. The first aggregator submit the proof will earn the right to create the validity proof of new state of the Layer 2 - -![](assets/polygon-zkevm-architecture_6066873078dcd11f9ef93601eba9237c52cbf11a.webp) - -This mechanism will avoid control of a single party and many of the potential attacks, since any Sequencer can propose a batch, but there is a cost on it. - -#### How to incentivization for Sequencer and Aggregators? - -The two permissionless participants of the zkEVM network are: Sequencers and Aggregators. Proper incentive structures have been devised to keep the zkEVM network fast and secure. Below is a summary of the fee structure for Sequencers and Aggregators: - -- **Sequencer** - - Collect transactions and publish them in a batch - - Receive fees from the published transactions - - Pay L1 transaction fees + MATIC (depends on pending batches) - - MATIC goes to Aggregators - - Profitable if: txs fees > L1 call + MATIC fee -- **Aggregator?** - - Process transactions published by Sequencers - - Build zkProof - - Receive MATIC from Sequencer - - Static Cost: L1 call cost + Server cost (to build a proof) - - Profitable if: MATIC fee > L1 call + Server cost - -#### zkEVM - -zkEVM was design to take advantage of ZK folklore to minimize size validity proof for validation, reduce transaction finality time and save gas costs. - -![](assets/polygon-zkevm-architecture_polygon-zk-prover-design-approach.webp) - -#### zkProver - -Have 4 main components: - -- The Executor, which is the Main State Machine Executor -- The STARK Recursion Component -- The CIRCOM Library -- The zk-SNARK Prover - -Prover generate verifiable proof process: - -![](assets/polygon-zkevm-architecture_fig-main-prts-zkpr.webp) - -You can read more [here](https://docs.hermez.io/zkEVM/zkProver/Overview/zkProver-Overview/#the-stark-recursion-component) - -#### Bridge flow - -**The Bridge L1 contract** have two operations, it requires two Merkle trees in order to work: globalExitTree and mainnet exit tree. - -- **bridge** transfer asset from one rollup to another -- **claim** make claim from any rollup - -**The Bridge L2 contract** named the global exit root manager L2 is responsible for managing the exit roots across multiple networks. - -#### RPC - -Provide a RPC interface compatible with ethereum so application like Metamask, etherscan can connect and interact. RPC also add transactions o the pool and interact with the state via read-only methods. - -#### State - -State implement a Merkle Tree and connect to DB backend. it checks integrity of block, transaction information. State also stores smart contract code in to the merkle tree and process transaction using EVM. - -## Reference - -- [Polygon zkEVM documentation](https://docs.hermez.io/zkEVM/Basic-Concepts/Intro-zkProver%27s-Design-Approach/) -- [Proof of efficiency](https://ethresear.ch/t/proof-of-efficiency-a-new-consensus-mechanism-for-zk-rollups/11988) -- [LX-to-LY bridge](https://wiki.polygon.technology/docs/zkEVM/lx-ly-bridge) -- [zkEvm](https://wiki.polygon.technology/docs/zkEVM/proof-of-efficiency) -- [Repo zkevm](https://github.com/0xPolygonHermez/zkevm-node) -- [Polygon zkEVM](https://mirror.xyz/msfew.eth/JJudP_Kf-IS6VhbF-qU0BUor1Ap6SFEb0TzYOHZ34Rc) diff --git a/topics/blockchain/precautions.md b/topics/blockchain/precautions.md deleted file mode 100644 index 692d7d494..000000000 --- a/topics/blockchain/precautions.md +++ /dev/null @@ -1,383 +0,0 @@ ---- -title: Precautions -description: Learn how to prepare smart contracts for failure with techniques like circuit breakers, rate limits, speed bumps, and upgradeability to enhance blockchain security and reduce risks. -date: null -redirect: - - /y1jyKw ---- - -As we discussed in the General Philosophy section, it is not enough to protect yourself against the known attacks. Since the cost of failure on a blockchain can be very high, you must also adapt the way you write software, to account for that risk. - -The approach we advocate is to "prepare for failure". It is impossible to know in advance whether your code is secure. However, you can architect your contracts in a way that allows them to fail gracefully, and with minimal damage. This section presents a variety of techniques that will help you prepare for failure. - -Note: There's always a risk when you add a new component to your system. A badly designed fail-safe could itself become a vulnerability - as can the interaction between a number of well-designed fail-safes. Be thoughtful about each technique you use in your contracts, and consider carefully how they work together to create a robust system. - -## Deployment - -Contracts should have a substantial and prolonged testing period - before substantial money is put -at risk. - -At minimum, you should: - -- Have a full test suite with 100% test coverage (or close to it) -- Deploy on your own testnet -- Deploy on the public testnet with substantial testing and bug bounties -- Exhaustive testing should allow various players to interact with the contract at volume -- Deploy on the mainnet in beta, with limits to the amount at risk - -##### Automatic deprecation - -During testing, you can force an automatic deprecation by preventing any actions, after a certain -time period. For example, an alpha contract may work for several weeks and then automatically shut -down all actions, except for the final withdrawal. - -```sol -modifier isActive() { - require(block.number <= SOME_BLOCK_NUMBER); - _; -} - -function deposit() public isActive { - // some code -} - -function withdraw() public { - // some code -} -``` - -##### Restrict amount of ether per user/contract - -In the early stages, you can restrict the amount of Ether for any user (or for the entire contract) -\- reducing the risk. - -## Circuit breakers - -Circuit breakers stop execution if certain conditions are met, and can be useful when new errors -are discovered. For example, most actions may be suspended in a contract if a bug is discovered, -and the only action now active is a withdrawal. You can either give certain trusted parties the -ability to trigger the circuit breaker or else have programmatic rules that automatically trigger -the certain breaker when certain conditions are met. - -Example: - -```sol -bool private stopped = false; -address private owner; - -modifier isAdmin() { - require(msg.sender == owner); - _; -} - -function toggleContractActive() isAdmin public { - // You can add an additional modifier that restricts stopping a contract to be based on another action, such as a vote of users - stopped = !stopped; -} - -modifier stopInEmergency { if (!stopped) _; } -modifier onlyInEmergency { if (stopped) _; } - -function deposit() stopInEmergency public { - // some code -} - -function withdraw() onlyInEmergency public { - // some code -} -``` - -## Rate limit - -Rate limiting halts or requires approval for substantial changes. For example, a depositor may only -be allowed to withdraw a certain amount or percentage of total deposits over a certain time period -(e.g., max 100 ether over 1 day) - additional withdrawals in that time period may fail or require -some sort of special approval. Or the rate limit could be at the contract level, with only a -certain amount of tokens issued by the contract over a time period. - -Example: - -```sol -uint internal period; // how many blocks before limit resets -uint internal limit; // max ether to withdraw per period -uint internal currentPeriodEnd; // block which the current period ends at -uint internal currentPeriodAmount; // amount already withdrawn this period - -constructor(uint _period, uint _limit) public { - period = _period; - limit = _limit; - - currentPeriodEnd = block.number + period; -} - -function withdraw(uint amount) public { - // Update period before proceeding - updatePeriod(); - - // Prevent overflow - uint totalAmount = currentPeriodAmount + amount; - require(totalAmount >= currentPeriodAmount, 'overflow'); - - // Disallow withdraws that exceed current rate limit - require(currentPeriodAmount + amount < limit, 'exceeds period limit'); - currentPeriodAmount += amount; - msg.sender.transfer(amount); -} - -function updatePeriod() internal { - if(currentPeriodEnd < block.number) { - currentPeriodEnd = block.number + period; - currentPeriodAmount = 0; - } -} -``` - -Some tips for running bounty programs: - -- Decide which currency bounties will be distributed in (BTC and/or ETH) -- Decide on an estimated total budget for bounty rewards -- From the budget, determine three tiers of rewards: - - smallest reward you are willing to give out - - highest reward that's usually awardable - - an extra range to be awarded in case of very severe vulnerabilities -- Determine who the bounty judges are (3 may be ideal typically) -- Lead developer should probably be one of the bounty judges -- When a bug report is received, the lead developer, with advice from judges, should evaluate the - severity of the bug -- Work at this stage should be in a private repo, and the issue filed on Github -- If it's a bug that should be fixed, in the private repo, a developer should write a test case, - which should fail and thus confirm the bug -- Developer should implement the fix and ensure the test now passes; writing additional tests as - needed -- Show the bounty hunter the fix; merge the fix back to the public repo is one way -- Determine if bounty hunter has any other feedback about the fix -- Bounty judges determine the size of the reward, based on their evaluation of both the - _likelihood_ and _impact_ of the bug. -- Keep bounty participants informed throughout the process, and then strive to avoid delays in - sending them their reward - -For an example of the three tiers of rewards, see -[Ethereum's Bounty Program](https://bounty.ethereum.org): - -> The value of rewards paid out will vary depending on severity of impact. Rewards for minor -> 'harmless' bugs start at 0.05 BTC. Major bugs, for example leading to consensus issues, will be -> rewarded up to 5 BTC. Much higher rewards are possible (up to 25 BTC) in case of very severe -> vulnerabilities. - -## Speed bumps - -Speed bumps slow down actions, so that if malicious actions occur, there is time to recover. For -example, [The DAO](https://github.com/slockit/DAO/) required 27 days between a successful request -to split the DAO and the ability to do so. This ensured the funds were kept within the contract, -increasing the likelihood of recovery. In the case of the DAO, there was no effective action that -could be taken during the time given by the speed bump, but in combination with our other -techniques, they can be quite effective. - -Example: - -```sol -struct RequestedWithdrawal { - uint amount; - uint time; -} - -mapping (address => uint) private balances; -mapping (address => RequestedWithdrawal) private requestedWithdrawals; -uint constant withdrawalWaitPeriod = 28 days; // 4 weeks - -function requestWithdrawal() public { - if (balances[msg.sender] > 0) { - uint amountToWithdraw = balances[msg.sender]; - balances[msg.sender] = 0; // for simplicity, we withdraw everything; - // presumably, the deposit function prevents new deposits when withdrawals are in progress - - requestedWithdrawals[msg.sender] = RequestedWithdrawal({ - amount: amountToWithdraw, - time: block.timestamp - }); - } -} - -function withdraw() public { - if(requestedWithdrawals[msg.sender].amount > 0 && block.timestamp > requestedWithdrawals[msg.sender].time + withdrawalWaitPeriod) { - uint amountToWithdraw = requestedWithdrawals[msg.sender].amount; - requestedWithdrawals[msg.sender].amount = 0; - - require(msg.sender.send(amountToWithdraw)); - } -} -``` - -## Upgradeability - -!!! warning -Smart Contract upgradeability is an active area of research. There are many important -questions, and risks related to smart contract upgradeability. Do your research into the state of -the art. We welcome discussion on the -[related issue](https://github.com/ConsenSys/smart-contract-best-practices/issues/164). - -Code will need to be changed if errors are discovered or if improvements need to be made. It is no -good to discover a bug, but have no way to deal with it. - -Designing an effective upgrade system for smart contracts is an area of active research, and we -won't be able to cover all of the complications in this document. However, two basic approaches are -most commonly used. The simpler of the two is to have a registry contract that holds the address of -the latest version of the contract. A more seamless approach for contract users is to have a -contract that forwards calls and data onto the latest version of the contract. - -Whatever the technique, it's important to have modularization and good separation between -components, so that code changes do not break functionality, orphan data, or require substantial -costs to port. In particular, it is usually beneficial to separate complex logic from your data -storage, so that you do not have to recreate all of the data in order to change the functionality. - -It's also critical to have a secure way for parties to decide to upgrade the code. Depending on -your contract, code changes may need to be approved by a single trusted party, a group of members, -or a vote of the full set of stakeholders. If this process can take some time, you will want to -consider if there are other ways to react more quickly in case of an attack, such as an -emergency stop or circuit-breaker. - -Regardless of your approach, it is important to have some way to upgrade your contracts, or they -will become unusable when the inevitable bugs are discovered in them. - -#### Example 1: Use a registry contract to store the latest version of a contract - -In this example, the calls aren't forwarded, so users should fetch the current address each time -before interacting with it. - -```sol -pragma solidity ^0.5.0; - -contract SomeRegister { - address backendContract; - address[] previousBackends; - address owner; - - constructor() { - owner = msg.sender; - } - - modifier onlyOwner() { - require(msg.sender == owner) - _; - } - - function changeBackend(address newBackend) public - onlyOwner() - returns (bool) - { - if(newBackend != address(0) && newBackend != backendContract) { - previousBackends.push(backendContract); - backendContract = newBackend; - return true; - } - - return false; - } -} -``` - -There are two main disadvantages to this approach: - -1. Users must always look up the current address, and anyone who fails to do so risks using an old - version of the contract -2. You will need to think carefully about how to deal with the contract data when you replace the - contract - -The alternate approach is to have a contract forward calls and data to the latest version of the -contract: - -#### Example 2: [Use a `DELEGATECALL`](http://ethereum.stackexchange.com/questions/2404/upgradeable-contracts) to forward data and calls - -This approach relies on using the -[fallback function](https://solidity.readthedocs.io/en/latest/contracts.html#fallback-function) (in -`Relay` contract) to forward the calls to a target contract (`LogicContract`) using -[delegatecall](https://solidity.readthedocs.io/en/latest/introduction-to-smart-contracts.html#delegatecall-callcode-and-libraries). -Remember that `delegatecall` is a special function in Solidity that executes the logic of the -called address (`LogicContract`) in the context of the calling contract (`Relay`), so _"storage, -current address and balance still refer to the calling contract , only the code is taken from the -called address"_. - -```sol -pragma solidity ^0.5.0; - -contract Relay { - address public currentVersion; - address public owner; - - modifier onlyOwner() { - require(msg.sender == owner); - _; - } - - constructor(address initAddr) { - require(initAddr != address(0)); - currentVersion = initAddr; - owner = msg.sender; // this owner may be another contract with multisig, not a single contract owner - } - - function changeContract(address newVersion) public - onlyOwner() - { - require(newVersion != address(0)); - currentVersion = newVersion; - } - - fallback() external payable { - (bool success, ) = address(currentVersion).delegatecall(msg.data); - require(success); - } -} -``` - -```sol -contract LogicContract { - address public currentVersion; - address public owner; - uint public counter; - - function incrementCounter() { - counter++; - } -} -``` - -This simple version of the pattern cannot return values from `LogicContract`'s functions, only -forward them, which limits its applicability. More complex implementations attempt to solve this -with in-line assembly code and a registry of return sizes. They are commonly referred to as -[Proxy Patterns](https://blog.openzeppelin.com/proxy-patterns/), but are also known as -[Router](https://github.com/PeterBorah/ether-router), -[Dispatcher](https://gist.github.com/Arachnid/4ca9da48d51e23e5cfe0f0e14dd6318f) and Relay. Each -implementation variant introduces a different set of complexity, risks and limitations. - -You must be extremely careful with how you store data with this method. If your new contract has a -different storage layout than the first, your data may end up corrupted. When using more complex -implementations of `delegatecall`, you should carefully consider and understand\*: - -- How the EVM handles the - [layout of state variables in storage](https://docs.soliditylang.org/en/latest/internals/layout_in_storage.html), - including packing multiple variables into a single storage slot if possible -- How and why - [the order of inheritance](https://github.com/OpenZeppelin/openzeppelin-sdk/issues/37) impacts - the storage layout -- Why the called contract (`LogicContract`) must have the same storage layout of the calling - contract (`Relay`), and only append new variables to the storage (see - [Background on delegatecall](https://blog.trailofbits.com/2018/09/05/contract-upgrade-anti-patterns/)) -- Why a new version of the called contract (`LogicContract`) - [must have the same storage layout as the previous version](https://github.com/OpenZeppelin/openzeppelin-sdk/issues/37), - and only append new variables to the storage -- [How a contract's constructor can affect upgradeability](https://blog.openzeppelin.com/towards-frictionless-upgradeability/) -- How the ABI specifies - [function selectors](https://solidity.readthedocs.io/en/v0.4.24/abi-spec.html?highlight=signature#function-selector) - and how - [function-name collision](https://medium.com/nomic-labs-blog/malicious-backdoors-in-ethereum-proxies-62629adf3357) - can be used to exploit a contract that uses `delegatecall` -- How `delegatecall` to a non-existent contract will return true even if the called contract does - not exist. For more details see - [Breaking the proxy pattern](https://blog.trailofbits.com/2018/09/05/contract-upgrade-anti-patterns/) - and Solidity docs on - [Error handling](https://solidity.readthedocs.io/en/latest/control-structures.html#error-handling-assert-require-revert-and-exceptions). -- Remember the - [importance of immutability to achieve truslessness](https://diligence.consensys.net/blog/2019/01/upgradeability-is-a-bug/) - -\* _Extended from -[Proxy pattern recommendations section](https://blog.trailofbits.com/2018/09/05/contract-upgrade-anti-patterns/)_ diff --git a/topics/blockchain/starknet-architecture.md b/topics/blockchain/starknet-architecture.md deleted file mode 100644 index 6afbd98df..000000000 --- a/topics/blockchain/starknet-architecture.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: StarkNet architecture -description: "This article provides an overview of StarkNet's architecture, including its main components, how it works, and its transaction lifecycle. It also covers the messaging mechanism between Layer 1 and Layer 2, as well as the node clients used in the StarkNet network." -date: 2022-12-26 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - evm - - stark-net - - zk-rollup -redirect: - - /KqI5Xg ---- - -## StarkNet layer 2 solution - -StarkNet is a layer 2 blockchain solution using ZK rollup, it provides StarkDex technology for well-known applications such as dYdX, ImmutableX, Sorare. It allows decentralized exchanges to process transactions with fast speed and low costs. Its essence is to reduce computations, store on-chain, replace with off-chain computations and store off-chain, Store balance using merkle tree with root merkle tree stored on-chain - -![](assets/starknet-architecture_starkdex.webp) - -## StarkNet architecture overview - -The system consists of 6 main components: - -- **User account** is a smart contract and expands the ability to create a recovery mechanism that depends on social information such as friends, family, colleagues, can handle offline authentication instead of use seed phrases. There is now an Argent X wallet that uses this method. -- **Sequencer** validates off-chain transactions, manages orders, verifies and bundles transactions into blocks. The system has only 1 sequencer to ensure it works consistently. It also uses a virtual machine similar to EVM called Cairo -- **Prover** generates proof to verify transactions wrapped by sequencer to generate global state by processing transactions in new block. To generate a valid proof it requires an execution trace of the Sequencer's computations. Prover generates proof for all other applications running on StarkEx. -- **Full node** is a component that keeps a record of all transactions made during the rollup and tracks the global state of the network. They communicate p2p sharing information about the global state and validating every time a new block is created. -- **Verifier** is a smart contract running on Layer 1 Ethereum that is responsible for verifying on-chain proofs generated by Prover and transactions on Layer 1. Verification results are sent to smart contract StarkNet core for storage. and mark the start of a new set of transactions on Layer 1 from StarkNet to update the Global state on-chain. -- **StarkNet core** Is a smart contract running on layer 1 that receives changes to Layer 2 global state from StarkNet every time there is a new L2 block and its proof is successfully verified on-chain by Verifier. StarkNet Fullnode will decrypt the data in the "call data" to recreate the history of the network on the first sync - -## How does it work - -The process consists of four steps: - -1. **Batching** is Sequencer, groups together multiple transactions into a batch for processing. The entired batch is submit on-chain as a single compressed state update with a proof. -2. **Validating & updating** The update is then compressed in the form of a hash on the entire state of the system : ℎ(ℎ(ℎ(class_hash,storage_root),0),0) Where: - -- class_hash is the hash of the contract’s definition discussed here -- storage_root is the root of another Merkle-Patricia tree of height 251 that is constructed from the contract’s storage -- ℎ is the Pedersen hash function. - -3. **Generating a proof** Once the batch transaction is processed, StarkEx generates a STARK proof to confirm the correctness of the transactions. -4. **On-chain verification** Once the proof is verified, the state update is committed and settled on layer 1 Ethereum - -## Messaging mechanism - -Contracts on L2 can interact asynchronously with contracts on L1 via the L2→L1 messaging protocol. - -![](assets/starknet-architecture_starknet-current-architecture.webp) - -Contracts on L1 can interact asynchronously with contracts on L2 via the L1→L2 messaging protocol. The protocol consists of the following stages: - -![](assets/starknet-architecture_starknet-l1l2.webp) - -## StarkNet transaction lifecycle - -When the transaction is submitted to the StarkNet, it is sent to the Sequence node. **Sequencer** takes a batch of transactions and generates: - -- List of changes made by transactions (storage, balance, data...) -- As a proof, if every transaction in the batch is successfully processed compared to the previous state of the network then the result will be the list of changes listed previously. - -![](assets/starknet-architecture_starknet-transactions-states.webp) - -## Node clients - -StarkNet nodes use the [Pathfinder](https://github.com/eqlabs/pathfinder) or the [Juno](https://github.com/NethermindEth/juno) client and they are similar to the nodes running Go Ethereum - -## Reference - -- [StarkDEX deep dive : introduction](https://medium.com/starkware/starkdex-deep-dive-introduction-7b4ef0dedba8) -- [StarkNet’s architecture review](https://david-barreto.com/StarkNets-architecture-review/) -- [StarkNet docs](https://docs.StarkNet.io/documentation/) - -### Smart contract - -- [StarkNet core](https://etherscan.io/address/0xc662c410c0ecf747543f5ba90660f6abebd9c8c4) -- [StarkNet operator](https://etherscan.io/address/0x2c169dfe5fbba12957bdd0ba47d9cedbfe260ca7) diff --git a/topics/blockchain/token.md b/topics/blockchain/token.md deleted file mode 100644 index 21dc2265b..000000000 --- a/topics/blockchain/token.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: Token -description: Learn about frontrunning risks in EIP-20 token approvals and the importance of following accepted Ethereum token standards like EIP-20 and EIP-721 for secure smart contracts. -date: null -redirect: - - /kY1gvw ---- - -## Frontrunning - -The EIP-20 token's `approve()` function creates the potential for an approved spender to spend more -than the intended amount. A -front running attack can be -used, enabling an approved spender to call `transferFrom()` both before and after the call to -`approve()` is processed. More details are available on the -[EIP](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-20.md#approve), and in -[this document](https://docs.google.com/document/d/1YLPtQxZu1UAvO9cZ1O2RPXBbT0mooh4DYKjA_jp-RLM/edit). - -## Standardization - -Generally speaking, smart contracts of tokens should follow an accepted and stable standard. - -Examples of currently accepted standards include: - -- [EIP20](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-20.md) -- [EIP721](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md) (non-fungible token) -- More at [eips.ethereum.org](https://eips.ethereum.org/erc#final) diff --git a/topics/blockchain/using-foundry-for-evm-smart-contract-developement.md b/topics/blockchain/using-foundry-for-evm-smart-contract-developement.md deleted file mode 100644 index f12fd3a5a..000000000 --- a/topics/blockchain/using-foundry-for-evm-smart-contract-developement.md +++ /dev/null @@ -1,449 +0,0 @@ ---- -title: Using Foundry for EVM smart contract development -description: "Introduce Foundry's core functionalities & practices to develop, test and deploy EVM smart contracts" -date: 2024-07-19 -authors: - - haongo138 -tags: - - blockchain - - evm - - foundry -redirect: - - /lsQ8Wg ---- - -## Introduction - -Foundry is a blazing-fast, all-in-one toolkit built by developers and for developers. Crafted with the speed of Rust, Foundry streamlines your entire workflow: - -- Write, test, deploy and script all within a single, unified environment. -- Experience lighting-fast compilation and test execution that leaves traditional tools in the dust. -- Harness the power of native Solidity scripting for automation & streamlined interactions. - -To help everyone to adopt Foundry in your next projects and start forging the future of decentralized applications. This article will walk through the core concepts of Foundry. - -## Why choosing Foundry? - -**Streamlined Development Workflow:** - -- Provides a unified environment for the entire development lifecycle, from writing and testing contracts to deploying and interacting with them, all within a single toolchain -- Allows you to write scripts directly in Solidity, simplifying tasks like deployments, contract interactions, and automated testing. This eliminates the need for external scripting languages. -- Foundry's CLI (using the forge and cast commands) offers a powerful and efficient way to interact with the toolchain, making it easy to integrate into existing workflows and automation scripts. - -**Blazing fast performance:** Foundry is built with Rust, a language renowned for its performance. This translates to significantly faster compilation and test execution times compared to tools like Truffle or Hardhat, which rely on JavaScript. - -![](assets/using-foundry-for-evm-smart-contract-developement_22ed3c2228f0f9355fcb48a2c63788ee_md5.webp) - -## Managing dependencies - -Currently, there are two ways of managing dependencies in a Foundry projects - -- Using [Git Submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) -- Using [Soldeer](https://soldeer.xyz/) - A Solidity native dependency manager - -By default, Foundry uses _Git Submodules_ for managing dependencies. However, I prefer _Solder_ for my EVM repository template because it integrates well with _forge_ - A command-line tool included with Foundry. - -## Toolbox overview - -Foundry offers a powerful suite of tools to resolve our development development needs: - -- [Forge](https://book.getfoundry.sh/forge/) - Builds, tests and deploys EVM smart contracts -- [Cast](https://book.getfoundry.sh/cast/) - Allows interaction with smart contracts, including making calls, sending transactions, and retrieving data. -- [Anvil](https://book.getfoundry.sh/anvil/) - Create local testnet node for deploying and testing smart contracts -- [Chisel](https://book.getfoundry.sh/chisel) - Provides an advanced Solidity REPL for rapid testing of code snippets. - -In the scope of this memo, we'll focus on _Forge_ for constructing a template EVM contract repository. - -## Showcase preparation - -Let's create a fresh repository with the following files and folders: - -``` -- src - - contracts - - IcySwap.sol - - scripts - - IcySwap.s.sol - - test - - IcySwap.t.sol -- foundry.toml -``` - -Folder structure explaination: - -- src/contracts: Contains all the smart contracts -- src/scripts: Contains all the scripts to interact with the contracts -- src/test: Contains all the test cases -- foundry.toml: Contains the Foundry configurations - -### Configure Foundry - -To make this guide more practical, a step closer to Mainnet deployments, we'll work on [Base Sepolia testnet](https://sepolia.basescan.org). - -If you want to verify your contracts on `Basescan`, you must create a [Block Explorer API Key](https://docs.base.org/quick-start/block-explorer-api-key) and set it in `.env` file. - -``` -BLOCK_EXPLORER_API_KEY= -``` - -Then, insert below code into `foundry.toml` file - -```toml -[profile.default] -src = 'contracts' -out = 'out' -script = 'scripts' -libs = ['node_modules', 'dependencies'] -test = 'test' -cache_path = 'cache_forge' - -[rpc_endpoints] -base = "https://mainnet.base.org" -base_sepolia = "https://sepolia.base.org" -base_goerli = "https://goerli.base.org" - -[etherscan] -base = { key = "${BLOCK_EXPLORER_API_KEY}" } -base_sepolia = { key = "${BLOCK_EXPLORER_API_KEY}" } -base_goerli = { key = "${BLOCK_EXPLORER_API_KEY}" } -``` - -### Install dependencies - -_Make sure you have `Foundry toolchain` installed, if not, please follow [this guide](https://book.getfoundry.sh/getting-started/installation#using-foundryup) to install it._ - -**Soldeer** is a Solidity native package manager that helps us to manage dependencies in a more efficient way, just like using `npm` in Node.js. - -Libraries that we'll use in this tutorial: - -- [forge-std](https://v2.soldeer.xyz/project/forge-std) - a collection of helpful contracts and libraries used for writing tests or deployment scripts in native Solidity -- [@openzeppelin-contracts](https://v2.soldeer.xyz/project/@openzeppelin-contracts) - a library for secure smart contract development - [@openzeppelin-contracts-upgradeable](https://v2.soldeer.xyz/project/@openzeppelin-contracts-upgradeable) - contains upgradeable variant of OpenZeppelin Contracts - -**Steps to install above dependencies:** - -1/ Append the following code into existing `foundry.toml` file: - -```toml -[dependencies] -"@openzeppelin-contracts" = { version = "5.0.2", url = "https://soldeer-revisions.s3.amazonaws.com/@openzeppelin-contracts/5_0_2_14-03-2024_06:11:59_contracts.zip" } -"@openzeppelin-contracts-upgradeable" = { version = "5.0.2", url = "https://soldeer-revisions.s3.amazonaws.com/@openzeppelin-contracts-upgradeable/5_0_2_14-03-2024_06:12:07_contracts-upgradeable.zip" } -forge-std = { version = "1.9.1", url = "https://soldeer-revisions.s3.amazonaws.com/forge-std/v1_9_1_03-07-2024_14:44:59_forge-std-v1.9.1.zip" } -``` - -2/ Run `forge soldeer install` command to install libraries from `[dependencies]` section. - -3/ Replace all content in `remappings.txt` file with the following code: - -```bash -@openzeppelin/contracts=dependencies/@openzeppelin-contracts-5.0.2 -@openzeppelin-contracts-upgradeable=dependencies/@openzeppelin-contracts-upgradeable-5.0.2 -@forge-std=dependencies/forge-std-1.9.1 -``` - -### Prepare some contracts - -In this showcase, we'll utilize [IcySwap](https://github.com/dwarvesf/contract-icy-swap/blob/main/contracts/IcySwap.sol) contract, let's create a file called `IcySwap.sol` in `src/contracts` folder. - -```solidity -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; - -import "@openzeppelin/contracts/token/ERC20/IERC20.sol"; -import "@openzeppelin/contracts/token/ERC20/ERC20.sol"; -import "@openzeppelin/contracts/token/ERC20/utils/SafeERC20.sol"; -import "@openzeppelin/contracts/utils/Pausable.sol"; -import "@openzeppelin/contracts/utils/ReentrancyGuard.sol"; -import "@openzeppelin/contracts/access/Ownable.sol"; - -contract IcySwap is Ownable, Pausable, ReentrancyGuard { - using SafeERC20 for IERC20; - - IERC20 public immutable usdc; - IERC20 public immutable icy; - - // This conversion is follow usdc decimals: 10**6 - // Let say we want 1 icy equal 2 usdc -> conversion rate should be 2 * 10**6 - uint256 public icyToUsdcConversionRate; - - event Swap(IERC20 indexed fromToken, uint256 indexed fromAmount); - event ConversionRateChanged(uint256 conversionRate); - event WithdrawToOwner(IERC20 indexed token, uint256 amount); - - constructor(address initialOwner, IERC20 _usdc, IERC20 _icy, uint256 _conversionRate) - Ownable(initialOwner) - { - usdc = _usdc; - icy = _icy; - icyToUsdcConversionRate = _conversionRate; - } - - // Swap methods - function swap(uint256 _amountIn) external nonReentrant whenNotPaused { - uint256 amountOut = (_amountIn * icyToUsdcConversionRate) / (10 ** 18); - _swap(icy, _amountIn, usdc, amountOut); - emit Swap(icy, _amountIn); - } - - // Moderate methods - function setConversionRate(uint256 _conversionRate) external onlyOwner { - icyToUsdcConversionRate = _conversionRate; - emit ConversionRateChanged(_conversionRate); - } - - function pause() external onlyOwner { - _pause(); - } - - function unpause() external onlyOwner { - _unpause(); - } - - function withdrawToOwner(IERC20 _token) external onlyOwner { - uint256 balance = _token.balanceOf(address(this)); - require(balance > 0, "contract has no balance"); - _token.safeTransfer(msg.sender, balance); - emit WithdrawToOwner(_token, balance); - } - - // Internal methods - function _swap(IERC20 _fromToken, uint256 _fromAmount, IERC20 _toToken, uint256 _toAmount) - internal - { - require(_toToken.balanceOf(address(this)) >= _toAmount, "out of money"); - _fromToken.safeTransferFrom(msg.sender, address(this), _fromAmount); - _toToken.safeTransfer(msg.sender, _toAmount); - } -} -``` - -Next, we'll create 2 sample ERC20 tokens to test our swapping logic. - -- Create a file called `ICY.sol` in `src/contracts` folder. - -```solidity -//SPDX-License-Identifier: Unlicense -pragma solidity ^0.8.24; - -import "@openzeppelin/contracts/token/ERC20/ERC20.sol"; - -contract IcyToken is ERC20 { - uint256 constant _initial_supply = 1000000000 * (10 ** 18); - - constructor() ERC20("IcyToken", "ICY") { - _mint(msg.sender, _initial_supply); - } - - function mint(address to, uint256 amount) public { - _mint(to, amount); - } -} -``` - -- Create a file called `USDC.sol` in `src/contracts` folder. - -```solidity -//SPDX-License-Identifier: Unlicense -pragma solidity ^0.8.24; - -import "@openzeppelin/contracts/token/ERC20/ERC20.sol"; - -contract UsdcToken is ERC20 { - uint256 constant _initial_supply = 1000000000 * (10 ** 18); - - constructor() ERC20("UsdcToken", "USDC") { - _mint(msg.sender, _initial_supply); - } - - function mint(address to, uint256 amount) public { - _mint(to, amount); - } -} -``` - -## The fun part begins - -In this part, we'll use `forge-std` library which provides a set of helpful [Cheatcodes](https://book.getfoundry.sh/forge/cheatcodes) to: - -- Create test scripts to test `IcySwap` contract. -- Create deployment script to deploy `IcySwap` contract to `Base Sepolia` testnet. - -### Testing `IcySwap` contract - -Create a file called `IcySwap.t.sol` in `src/test` folder. - -```solidity -// SPDX-License-Identifier: UNLICENSED -pragma solidity ^0.8.24; - -import "@forge-std/src/Test.sol"; -import "@forge-std/src/Vm.sol"; -import "@forge-std/src/console2.sol"; -import "@openzeppelin/contracts/proxy/transparent/ProxyAdmin.sol"; -import "@openzeppelin/contracts/proxy/transparent/TransparentUpgradeableProxy.sol"; -import "@openzeppelin/contracts/utils/cryptography/EIP712.sol"; - -import "../contracts/IcySwap.sol"; -import "../contracts/ICY.sol"; -import "../contracts/USDC.sol"; - -contract StreamPointsTest is Test { - IcySwap internal icySwap; - IcyToken internal icy; - UsdcToken internal usdc; - - address internal user; - address internal icySwapOwner; - address internal ICY_ADDRESS; - address internal USDC_ADDRESS; - - function setUp() public virtual { - user = address(1); - icySwapOwner = address(2); - - icy = new IcyToken(); - usdc = new UsdcToken(); - - vm.startPrank(icySwapOwner); - icySwap = new IcySwap( - IERC20(address(usdc)), - IERC20(address(icy)), - 2 * 10 ** 6 - ); - vm.stopPrank(); - } - - function test_swap() public virtual { - vm.startPrank(user); - - // prepare balance - icy.mint(user, 150 * 10 ** 18); - usdc.mint(address(icySwap), 150000 * 10 ** 18); - icy.approve(address(icySwap), type(uint256).max); - - // start swap - icySwap.swap(100 * 10 ** 18); - - vm.stopPrank(); - assertEq( - icy.balanceOf(address(icySwap)), - 100 * 10 ** 18, - "failed to swap" - ); - } - - function test_setConversionRate() public virtual { - vm.prank(icySwapOwner); - icySwap.setConversionRate(3 * 10 ** 6); - assertEq( - icySwap.icyToUsdcConversionRate(), - 3 * 10 ** 6, - "failed to set conversion rate" - ); - } - - function test_withdrawToOwner() public virtual { - vm.startPrank(icySwapOwner); - - icy.mint(address(icySwap), 150 * 10 ** 18); - icySwap.withdrawToOwner(icy); - - vm.stopPrank(); - assertEq( - icySwap.icy().balanceOf(address(icySwap)), - 0, - "failed to withdraw to owner" - ); - } - - function test_RevertWhen_CallerIsNotOwner() public { - vm.expectRevert(); - vm.prank(user); - icySwap.setConversionRate(3 * 10 ** 6); - } -} -``` - -**Explaination** - -1/ In `setUp()` function, we deploy: - -- Use cheatcode `vm.startPrank(icySwapOwner)` to start a transaction from `icySwapOwner` account to deploy `IcySwap` contract. -- Deploy 2 sample ERC20 tokens `ICY` and `USDC` used for `ICY/USDC` pair swapping. - -2/ In each test function, we also use cheatcode `vm.startPrank(user)` to start a transaction from `user` account to interact with `IcySwap` contract. Then make assertions to check the expected results using provided methods from `forge-std` library. - -**Run the test** - -You can run the test with traces by using the following command: - -```bash -forge test -vvvv -``` - -`Traces` is a feature that allows you to see the internal calls of a transaction. It's useful for debugging and understanding how a contract works. -![](assets/using-foundry-for-evm-smart-contract-developement_a5553a471de0e3b37e81f94efdf3f1c6_md5.webp) - -### Deploying `IcySwap` contract to `Base Sepolia` testnet - -Before deploying `IcySwap` contract to `Base Sepolia` testnet, we need to have following environment variables in `.env` file: - -``` -WALLET_PRIVATE_KEY= -BLOCK_EXPLORER_API_KEY= -``` - -Create a file called `IcySwap.s.sol` in `src/scripts` folder with the following content: - -```solidity -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; - -import { Script, console2 } from "@forge-std/src/Script.sol"; -import "../contracts/IcySwap.sol"; - -contract IcySwapScript is Script { - function setUp() public { } - - function run() public { - uint256 privateKey = vm.envUint("WALLET_PRIVATE_KEY"); - address ICY_ADDRESS= 0x78a3f816a8e26af8c09F6Da3995Ee19bd69bf7fF; - address USDC_ADDRESS = 0x036CbD53842c5426634e7929541eC2318f3dCF7e; - vm.startBroadcast(privateKey); - - IcySwap icySwap = new IcySwap(IERC20(USDC_ADDRESS), IERC20(ICY_ADDRESS), 2 * 10**6); - - vm.stopBroadcast(); - console2.log("IcySwap address: ", address(icySwap)); - } -} -``` - -In above script: - -- We can use `vm.envUint()` to get the value of `WALLET_PRIVATE_KEY` environment variable. -- Then, use `vm.startBroadcast()` to start a transaction from the account with the private key to deploy `IcySwap` contract. - -Now, we can run the deployment script to deploy & verify our contract, by running: - -```bash -forge script scripts/IcySwap.s.sol:IcySwapScript --broadcast --verify --rpc-url base_sepolia -``` - -And our `IcySwap` contract will be deployed to `Base Sepolia` testnet & will automatically be verified. - -![](assets/using-foundry-for-evm-smart-contract-developement_edacb4045a35d14a161b41e829079199_md5.webp) - -## Other usages - -Foundry is not just be here to resolve our common tasks like compiling, testing and deploying smart contracts. It also provides a lot of other features that can be used to enhance our development workflow. One of them is `Fork testing`. - -[Fork testing](https://book.getfoundry.sh/forge/fork-testing) is like a time machine that allows us to test our contracts on a forked mainnet and move to a specific block for testing. - -I found an interesting repository that use Foundry to reproduce a lot of DeFi hacked incidents in the past - It's [DeFiHackLabs](https://github.com/SunWeb3Sec/DeFiHackLabs). - -## References - -- -- -- diff --git a/topics/blockchain/web3-development-with-foundry.md b/topics/blockchain/web3-development-with-foundry.md deleted file mode 100644 index 54f3c26ad..000000000 --- a/topics/blockchain/web3-development-with-foundry.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -title: Web3 development with Foundry -description: Provides a comprehensive guide to Web3 development using Foundry, a modern, Rust-based toolkit for Ethereum smart contract development. -date: 2025-04-01 -authors: - - haongo138 -toc: false -tags: - - blockchain - - evm - - foundry - - web3 -redirect: - - /BBc4Xw ---- - -## Overview of Foundry - -Foundry is a blazingly fast, portable, and modular toolkit for Ethereum application development written in Rust. It consists of three main components: - -- **Forge**: Testing framework for Ethereum smart contracts -- **Cast**: Swiss army knife for interacting with EVM smart contracts -- **Anvil**: Local Ethereum node designed for development - -![](assets/web3-development-with-foundry-00.jpg) - -## Why others not using Hardhat? - -Foundry's Rust-based architecture makes testing much faster than JavaScript alternatives. Security teams and auditors prefer working directly in Solidity without translation layers. The framework's adoption has grown quickly in 2024, especially for high-value contracts where performance and reliability matter. - -Foundry's terminal-based workflow cuts out JavaScript overhead, making it perfect for developers who want to work closer to the metal. Security teams love its deterministic environment when dealing with complex contracts. - -![](assets/web3-development-with-foundry-01.jpg) - -## Why we not using Hardhat? - -Hardhat's lack of ESM support in TypeScript projects forced us to use outdated CommonJS modules. Since our frontend and services already use ESM, this created unnecessary friction in our development workflow. Foundry's language-agnostic approach lets us maintain a consistent ESM-based architecture across our entire stack. - -## Core benefits of Foundry - -**Development Speed**: Foundry accelerates development through fast compilation, native Solidity testing, and quick feedback loops, with benchmarks showing it's consistently 1.5-11x faster than Hardhat and up to 335x faster than Dapptools. - -The platform offers **Modern Developer Experience** with built-in fuzzing that can run 10,000 tests in seconds to find edge cases, powerful debugging tools for precise error identification, and comprehensive gas optimization features that help create efficient contracts. - -For **Flexibility**, Foundry seamlessly integrates with existing toolchains while supporting multiple EVM chains through its comprehensive toolkit consisting of Forge (for testing), Cast (for contract interaction), and Anvil (local Ethereum node), making it adaptable to various project requirements and easily incorporated into CI/CD pipelines for automated testing and deployment. - -In our projects, we've seen these benefits firsthand. Our team uses Foundry's fast testing to catch issues early in development, while the native Solidity testing helps us write more accurate tests. The gas optimization features have helped us reduce deployment costs by up to 30% in some cases. We particularly value the deterministic environment when working on complex DeFi contracts where every gas optimization matters. - -## What we actually do? - -### Dealing with dependencies and remapping - -![](assets/web3-development-with-foundry-02.jpg) - -#### Git Submodules (Traditional Approach) - -```bash -forge install OpenZeppelin/openzeppelin-contracts --no-commit -git submodule update --init --recursive -``` - -#### Modern package management with Bun - -```bash -bun init -bun add -d @openzeppelin/contracts -``` - -Configure remappings in `remappings.txt`: - -```text:remappings.txt -@openzeppelin/=node_modules/@openzeppelin/ -ds-test/=lib/forge-std/lib/ds-test/src/ -forge-std/=lib/forge-std/src/ -``` - -### Deploying and testing a smart contract - -We'll build an upgradeable ERC-1155 contract for game items (GOLD, SILVER, SWORD, SHIELD) using Foundry. This example shows how to: - -- Implement and test smart contracts -- Set up deployment scripts -- Handle contract upgrades using the UUPS upgrade pattern - -#### Implement a basic ERC-1155 contract - -First, let's create an upgradeable ERC-1155 contract: - -```solidity:src/GameItems.sol -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.13; - -import "@openzeppelin/contracts-upgradeable/token/ERC1155/ERC1155Upgradeable.sol"; -import "@openzeppelin/contracts-upgradeable/access/OwnableUpgradeable.sol"; -import "@openzeppelin/contracts-upgradeable/proxy/utils/Initializable.sol"; -import "@openzeppelin/contracts-upgradeable/proxy/utils/UUPSUpgradeable.sol"; - -contract GameItems is Initializable, ERC1155Upgradeable, OwnableUpgradeable, UUPSUpgradeable { - // Item IDs - uint256 public constant GOLD = 0; - uint256 public constant SILVER = 1; - uint256 public constant SWORD = 2; - uint256 public constant SHIELD = 3; - - /// @custom:oz-upgrades-unsafe-allow constructor - constructor() { - _disableInitializers(); - } - - function initialize() public initializer { - __ERC1155_init("https://game.example/api/item/{id}.json"); - __Ownable_init(); - __UUPSUpgradeable_init(); - - // Mint initial items - _mint(msg.sender, GOLD, 10**18, ""); - _mint(msg.sender, SILVER, 10**27, ""); - _mint(msg.sender, SWORD, 1000, ""); - _mint(msg.sender, SHIELD, 1000, ""); - } - - function mint(address account, uint256 id, uint256 amount) - public - onlyOwner - { - _mint(account, id, amount, ""); - } - - function _authorizeUpgrade(address newImplementation) - internal - onlyOwner - override - {} -} -``` - -#### Writing tests for our contract - -Create comprehensive tests for the contract: - -```solidity:test/GameItems.t.sol -// SPDX-License-Identifier: UNLICENSED -pragma solidity ^0.8.13; - -import "forge-std/Test.sol"; -import "../src/GameItems.sol"; -import "@openzeppelin/contracts/proxy/ERC1967/ERC1967Proxy.sol"; - -contract GameItemsTest is Test { - GameItems public implementation; - GameItems public gameItems; - address public owner; - address public user1; - - function setUp() public { - owner = address(this); - user1 = address(0x1); - - // Deploy implementation - implementation = new GameItems(); - - // Deploy proxy - bytes memory initData = abi.encodeWithSelector( - GameItems.initialize.selector - ); - ERC1967Proxy proxy = new ERC1967Proxy( - address(implementation), - initData - ); - gameItems = GameItems(address(proxy)); - } - - function testInitialBalance() public { - assertEq(gameItems.balanceOf(owner, gameItems.GOLD()), 10**18); - assertEq(gameItems.balanceOf(owner, gameItems.SILVER()), 10**27); - assertEq(gameItems.balanceOf(owner, gameItems.SWORD()), 1000); - assertEq(gameItems.balanceOf(owner, gameItems.SHIELD()), 1000); - } - - function testMinting() public { - gameItems.mint(user1, gameItems.GOLD(), 100); - assertEq(gameItems.balanceOf(user1, gameItems.GOLD()), 100); - } - - function testFailMintingUnauthorized() public { - vm.prank(user1); - vm.expectRevert("Ownable: caller is not the owner"); - gameItems.mint(user1, gameItems.GOLD(), 100); - } - - function testBatchTransfer() public { - uint256[] memory ids = new uint256[](2); - ids[0] = gameItems.GOLD(); - ids[1] = gameItems.SILVER(); - - uint256[] memory amounts = new uint256[](2); - amounts[0] = 100; - amounts[1] = 200; - - gameItems.safeBatchTransferFrom( - owner, - user1, - ids, - amounts, - "" - ); - - assertEq(gameItems.balanceOf(user1, gameItems.GOLD()), 100); - assertEq(gameItems.balanceOf(user1, gameItems.SILVER()), 200); - } -} -``` - -#### Add a deployment script - -Create a deployment script that handles both the implementation and proxy deployment: - -```solidity:script/GameItems.s.sol -// SPDX-License-Identifier: UNLICENSED -pragma solidity ^0.8.13; - -import "forge-std/Script.sol"; -import "../src/GameItems.sol"; -import "@openzeppelin/contracts/proxy/ERC1967/ERC1967Proxy.sol"; - -contract GameItemsScript is Script { - function run() public { - uint256 deployerPrivateKey = vm.envUint("PRIVATE_KEY"); - - vm.startBroadcast(deployerPrivateKey); - - // Deploy implementation - GameItems implementation = new GameItems(); - - // Prepare initialization data - bytes memory initData = abi.encodeWithSelector( - GameItems.initialize.selector - ); - - // Deploy proxy - ERC1967Proxy proxy = new ERC1967Proxy( - address(implementation), - initData - ); - - // Log addresses - console.log("Implementation deployed to:", address(implementation)); - console.log("Proxy deployed to:", address(proxy)); - - vm.stopBroadcast(); - } -} -``` - -#### Run the deployment - -```bash -# Deploy to local network -forge script script/GameItems.s.sol --fork-url http://localhost:8545 --broadcast - -# Deploy to testnet (e.g., Sepolia) -forge script script/GameItems.s.sol \ - --rpc-url $SEPOLIA_RPC_URL \ - --broadcast \ - --verify \ - -vvvv -``` - -#### Contract lifecycle: from development to deployment - -![](assets/web3-development-with-foundry-03.png) - -## Limitations - -While Foundry shines in performance, it has its drawbacks. The lack of multi-network config files makes cross-chain deployments more tedious than Hardhat. The debugging tools, though functional, can't match Truffle's step-by-step debugger. We've also felt the smaller plugin ecosystem - you'll often need to build custom tooling that would be readily available in Hardhat. - -Writing tests in Solidity instead of JavaScript creates a steeper learning curve, especially for web developers on our team. The docs are improving but still leave gaps around advanced features, and community resources are still catching up to Hardhat's mature ecosystem. - -## Our assessment - -After months of wrestling with Hardhat's ESM limitations in our TypeScript stack, switching to Foundry was a game-changer. Sure, rewriting our JavaScript tests in Solidity took time, and we missed some familiar plugins. But the payoff was worth it - our test suite now runs in 40 seconds instead of 7 minutes. - -Writing tests in Solidity turned out to be a blessing in disguise. It eliminated translation errors and made our tests more precise. For teams ready to invest in learning Foundry, it offers a rock-solid foundation that pays off in both development speed and contract quality. diff --git a/topics/blockchain/zk-snarks.md b/topics/blockchain/zk-snarks.md deleted file mode 100644 index ab40aab5c..000000000 --- a/topics/blockchain/zk-snarks.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -title: zk-SNARKs -description: ZK-SNARKs is a type of zero-knowledge proof that allows one party to prove to another that a given statement is true, without revealing any additional information. This article provides an overview of zk-SNARKs, how they work, and their applications in blockchain technology. -date: 2022-12-09 -authors: - - baenv -github_id: vhbien000 -tags: - - blockchain - - ethereum - - zero-knowledge -redirect: - - /I8Ri1A ---- - -## What is this? - -- **ZK-SNARKs** stands for **Zero-Knowledge Succinct Non-Interactive Argument of Knowledges**. - -- Nowadays, we often hear this word when mentioning [[ZK-Rollups]]. But it is actually a **Privacy-enhancing technology**, and has a lot of applications that we will delve into in another post. - -## Decomposition - -- **ZK** aka **Zero-Knowledge** mean: - - Prove possession of certain information. - - Without revealing that information. - - **For example:** - - Given the hash of a random number. - - The prover could convince the verifier that a number with this hash value exists without revealing what it is. -- **Succinct**: - - Can be verified within a few milliseconds. - - no matter how long the statement is. -- **Non-interactive**: - - In the first version of ZK, the prover and verifier had to communicate repeatedly for multiple rounds. - - Now, by implementing this characteristic, the proof consists of a single message sent from the prover to the verifier. - -## Implementation by example - -### Example 1: Function C - -- You have a program denoted `C`. -- C have 2 input `C(x,w)`: - - `x` is the public stuff, that can be shared with anyone. - - `w` is the secret witness. -- The condition `C(x,w)===true`, means "Prover actually knows a secret witness `w` satisfied a statement related to `x`". -- As a prover, how can we prove that we know `w`, without sending `w` to the verifier to check with the statements? -- **Can map this example to the example of the hash function of the previous part. Let's do it, and go to the next example.** - -### Example 2: Bob, Alice, and hash - -- Bob is given a hash `H`. -- Alice is given the original string `S` satisfied the condition when hashing `S` by a hash function such as `SHA`, `H` is issued (aka `SHA(S) === H`). -- How can Alice prove to Bob know that she knows the `S`? -- Normally - - Alice needs to send `S` to Bob. - - Bob needs to hash again to check `SHA(S) === H`. -- But in this case, `S` is a secret witness, and must not send to any locations, how can Alice prove this statement to Bob? - - Solution: - - Alice need a **proof** to send to Bob. - - This **proof** can prove `SHA(S)===H` is true. Mapping to program `C`, with public `x` as `H` and private `w` as `S`, in other words, when this **proof** proves `C(x,w)===true`, it means Bob can confirm that Alice knows this `w` aka `S` satisfied `SHA(S)===H` without knowledge about `S`. - - Example of `C`: - - `function C(x, w) { return ( sha256(w) == x );}` - - **Note that `C(x,w)===true` is proved by the `proof`, not by itself. How can?** - -### Example 3: Implementation of zk-SNARKs in simple words - -- A **ZK-SNARK** consists of three functions `G, P, V` defined as follows: - - - **Key generator** aka `G` - - - `G(lambda, C) = pk,vk` - - `lambda`: secret parameter. **Noted this**. - - `C`: a program that proves that `prover` knows `w` by `C(x,w)=true`. Aka above `C` in **Example 1** and **Example 2**. - - `pk`: proving key - - `vk`: verification key - - `pk` and `vk` are public parameters that only need to be generated once for a given program `C`. - - Can assume that: - - **From `pk`, we can create a `proof` that can be verified by using `vk` in a pre-defined way.** - - - **Prover function** aka `P` - - - `P(pk,x,w) = prf` - - `pk`: proving key issued from `G` - - `x`: public input - - `w`: private witness - - `prf`: proof proves that prover knows `w` satisfy program `C`. - - - **Verifier function** aka `V` - - `V(vk,x,prf)=true/false` - - `vk`: verification key generated from `G` - - `x`: public input - - `prf`: proof generated from `P` - -- By above functions, `V(vk,x,prf)=true` means: - - - With the proof `prf` generated depends on a logic that includes `x`, `w` and `pk`. - - And `pk` is a pair with `vk` so they are closely related to each other. - - So when having `vk`, `x`, `prf`, by a pre-defined way, we can confirm that `C(x,w)===true`. - -- **Note that `lambda` can cause security issues when used in real work** - - - Reason: anyone who knows `lambda` can generate fake proofs - - Example: - - From any `C`, and known `lambda`, can find a pair `f_pk` and `f_vk` - - From `f_pk`, malicious actor can generate a `fake_prf` that represents `C(x,w)==true` when checking with `f_vk`. - - **Solution**: [[Multi-Party Ceremonies]] for **Trusted Setup** -> To build `lambda`. - -- Resolve the **Example 2**: - - - Bob uses `G` to generate key, send `pk` to Alice - - Alice gen `prf` -> Sent to Bob - - Bob verifies using `vk` - - - **Issue**: Bob can't be a prover because he holds the `Lambda`. - - A trusted independent group separate from Alice and Bob could run the generator and create the proving key pk and verification key vk in such a way that no one learns about lambda. - -### Example 4: In Ethereum [[ZK-Rollups]] - -- Can add the building blocks of the verification algorithm to Ethereum in the form of precompiled contracts. - - [[Layer 2]], run `G` to generate `pk` and `vk` - - [[Layer 2]], the operator use `pk` to generate `proof` - - [[Layer 1]], the verifier contract use `vk`, `proof` and public `input x` (can be state changes/Merkle root hash, bla bla ble ble) - - [[Layer 1]], if valid -> trigger transaction / append ZK blocks / etc. - -## References - -- https://consensys.net/blog/developers/introduction-to-zk-snarks/ -- https://ethereum.org/en/developers/docs/scaling/zk-rollups/ -- https://vitalik.eth.limo/general/2022/06/15/using_snarks.html diff --git "a/topics/blockchain/\302\266 Blocks.md" "b/topics/blockchain/\302\266 Blocks.md" deleted file mode 100644 index 12ddbde4b..000000000 --- "a/topics/blockchain/\302\266 Blocks.md" +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Blocks -description: This article provides an overview of blocks in blockchain technology, including their structure, types, and how they work. -date: 2022-06-16 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain -redirect: - - /8DodyQ ---- - -## Blocks - -![](assets/blocks_lzrylgx.webp) - -### 1. Define block in the blockchain - -A block is thus a permanent store of records that, once written, cannot be altered or removed. A Block has a limited size and transaction to avoid All Block are verified on the blockchain by all nodes and stored on the blockchain Block data are batches of transactions with a hash of the previous block in the chain. This links blocks together (in a chain) because hashes are cryptographically derived from the block data. This prevents fraud, because one change in any block in history would invalidate all the following blocks as all subsequent hashes would change and everyone running the blockchain would notice. - -![](assets/blocks_svihd1p.webp) - -### 2. Block data - -#### 2.1 Standard data - -- Block height – The block number and length of the blockchain (in blocks) on creation of the current block. -- Timestamp – The time at which a miner mined the block. -- Transactions – The number of transactions included within the block. -- Miner – The address of the miner who mined the block. -- Reward – The amount of ETH awarded to the miner for adding the block (standard 2ETH reward + any transaction fees of transactions included in the block). -- Difficulty – The difficulty associated with mining the block. -- Size – The size of the data within the block (measured in bytes). -- Gas used – The total units of gas used by the transactions in the block. -- Gas limit – The total gas limits set by the transactions in the block. -- Extra data – Any extra data the miner has included in the block. - -#### 2.2 Advanced data - -- Hash - The cryptographic hash that represents the block header (the unique identifier of the block). -- Parent hash – The hash of the block that came before the current block. -- Sha3Uncles – The combined hash of all uncles for a given parent. -- StateRoot – The root hash of Merkle trie which stores the entire state of the system. -- Nonce – A value used to demonstrate proof-of-work for a block by the miner. - -#### 2.3 Uncle blocks - -Uncle blocks are created when two miners create blocks at near-enough the same time – only one block can be validated across the nodes. They are not included but still receive a reward for the work. - -Block explorers provide information about uncle blocks like: - -- An uncle block number. -- A time they occurred. -- The block height at which they were created. -- Who mined it. -- The ETH reward. - -![](assets/blocks_eqhpghw.webp) - -### 3. block time - -Block time refers to the time it takes to mine a new block. In Ethereum, the average block time is between 12 to 14 seconds and is evaluated after each block. The expected block time is set as a constant at the protocol level and is used to protect the network's security when the miners add more computational power. The average block time gets compared with the expected block time, and if the average block time is higher, then the difficulty is decreased in the block header. If the average block time is smaller, then the difficulty in the block header will be increased. - -A new block can be rejected, please be careful with the new block, You need to wait for maximum node to verify this block before use. - -### 4. block size - -A final important note is that blocks themselves are bounded in size. Each block has a target size of 15 million gas but the size of blocks will increase or decrease in accordance with network demands, up until the block limit of 30 million gas (2x target block size). The total amount of gas extended by all transactions in the block must be less than the block gas limit. This is important because it ensures that blocks can’t be arbitrarily large. If blocks could be arbitrarily large, then less performant full nodes would stop being able to keep up with the network due to space and speed requirements. - -### 5. mining's relationship to blocks - -Mining is the term used for solving the number that is the nonce, the only number that can be changed in a block header. It is also the process the cryptocurrency's network uses if proof-of-work is used in the protocol. - -![](assets/blocks_qi5jtrd.png.webp) - -Cryptocurrency mining is commonly thought to be a complex mathematical problem; it is actually a random number generated through hashing. Hashing is the process of encrypting information using the encryption method a cryptocurrency uses. For example, Bitcoin uses SHA256 for its encryption algorithm. For a miner to generate the "winning" number, the mining program must use SHA 256 to hash random numbers and place them into the nonce to see if it is a match. - -### 6 Gas in block - -Gas is used to estimate the difficulty of all transactions in the block. Every function in a smart contract or transaction on blockchain will pay gas to process. To submit a transaction and avoid miners delaying your transaction forever, you need to pay enough gas Gas = Gas used \* Gas ​​price - -#### Reference - -https://ethereum.org/en/developers/docs/blocks/ diff --git "a/topics/blockchain/\302\266 Distributed systems.md" "b/topics/blockchain/\302\266 Distributed systems.md" deleted file mode 100644 index a79d5517e..000000000 --- "a/topics/blockchain/\302\266 Distributed systems.md" +++ /dev/null @@ -1,76 +0,0 @@ ---- -title: Distributed systems -description: This article provides an overview of distributed systems in blockchain technology, including their definition, advantages, and how they work. -date: 2022-06-16 -authors: - - thanhpn -github_id: thanhpn -tags: - - distributed-system - - blockchain - - foundational-topics -redirect: - - /Lud4qA ---- - -![](assets/blockchain-bridge_blockruption-blockchain-300h.webp) - -## Distributed systems on blockchain - -As defined on Wiki, distributed computing is a branch of computer science that studies distributed systems. A distributed system is a software system whose components are located on different computers, connected in a network. These computers work together as a single entity to accomplish a common task by exchanging messages. Distributed computing is decentralized parallel computing. The types of hardware, programming languages, operating systems, and other resources used can vary widely. It is similar to computer clustering with the main difference being the geographical distribution of resources. - -Distributed systems appear to be very common in practice. Most applications today, especially Internet applications, are implemented as Distributed systems. Deploying software, especially large systems, on multiple computing units (instead of using a single computer) has many benefits, such as: - -- Provide more resources when the system needs to handle a larger amount of work. -- Using only one computer unit means the risk of software crashing (crash) if that machine has a problem. Using multiple machine units will allow you to continue operating the software even if problems occur. -- When your system becomes complex and requires the combination of many different components, using DS allows you to break down a large system into many small units. Each unit can operate independently, can even be developed by different teams with different expertise. - -System users may be geographically dispersed across the globe. To ensure quality of service and limit latency, the machine system also needs to be distributed so that it can be as close to the user as possible. - -## Advantages of distributed systems: - -- Scalability: There are two types of scaling: horizontal scaling and vertical scaling: -- Reliability: Reliability is the fault tolerance of the system which means that the system will continue to provide its service as soon as one or more components (software/hardware) of the system fail. -- Availability: is the total time a system remains in normal operation for a specific period of time. A measure of availability is the percentage of time the system is up and running continuously for a period of time (usually 1 year). -- Efficiency: The efficiency of a distributed system is high load and low latency. This means that a system that can handle many concurrent requests with low latency is a high-performance system. -- Manageability: it is the ability to easily expand and maintain the system. In other words, it is the time to perform repair (repair) or maintain (maintain) when needed, the higher the time, the lower the availability. - -## Machine failure (node failure) - -Each physical computer, due to various reasons, can experience problems while in operation. These incidents are divided into several main categories: - -- Fail-stop: This is a type of problem that causes the process on the machine to stop working (stops computation as well as communication). The cause of this problem can be due to the machine crashing (software error, operating system error ...), hardware failure, or external causes (eg power failure). This is the most common type of problem, so when people talk about 'failure' without saying anything else, it's usually implied as this type of problem. Most of the algorithms developed in DS are intended to deal with this type of problem. -- Fail-recover: Process may be down for a certain time, but then recovery works again. The cause of this type of problem can be due to the machine rebooting automatically for some reason. Often when talking about this type of failure, people assume that the machine has the ability to store information on the hard drive and recover this information after the failure occurs. -- Byzantine failure: The problem that the computer does not work according to the requirements set forth. For example, the machine can send arbitrary messages, or change state arbitrarily, unlike what is programmed. This is the most annoying type of problem, which can happen when the system crashes for no apparent reason (e.g. RAM may be damaged causing bit-flip), or because the system is attacked by malicious actors. . - -## Network problems - -A computer network is also a physical product and so problems can also occur. A common type of problem is the “network partitioning” problem, which is simulated by the figure above. This problem occurs when the transmission of one or more servers is cut off from the rest of the system, causing the system to be split into many parts that cannot communicate with each other. In fact, in data centers, a cluster of servers is usually connected by one or more switches. Failure of the switch port or wire can lead to one or more servers being disconnected, leading to the aforementioned partitioning situation. - -## Distributed systems in blockchain - -![](assets/distributed-systems_c7xyhh1.webp) Blockchain is a distributed ledger, which simply means that a ledger is spread across the network among all peers (nodes) in the network. Every node has a copy of the Blockchain and once a block reaches a certain number of approved transactions then a new block is formed - -Any computer can join the blockchain network and become a validator by connecting to the internet and launching applications. The difference is that to become a node on the network, a computer needs to meet certain requirements in terms of connection speed, storage speed, and storage space. For the current bitcoin network, due to too many miners participating in the operation, the network requires a huge amount of computing power, If you want to participate in the network you need to run the service through the mines to contribute strength. - -## Distributed software on blockchain - -"The Ethereum Virtual Machine (EVM) is the runtime environment for transaction execution in Ethereum. It includes a stack, memory, gas balance (see below), program counter, and the persistent storage for all accounts (including contract code). When a transaction calls a contract's function, the arguments in the call are added to the stack and the EVM translates the contract's bytecode into stack operations. Stack items may be stored in memory or storage, and data from memory/storage may be added to the stack. The EVM is isolated from the other files and processes on the node's computer to ensure that for a given pre-transaction state and transaction, every node produces the same post-transaction state, thus enabling network consensus such as PoS. The formal definition of the EVM is specified in the Ethereum Yellow Paper. EVMs have been implemented in C++, C#, Go, Haskell, Java, JavaScript, Python, Ruby, Rust, Elixir, Erlang, and soon WebAssembly." - -The EVM's instruction set is Turing-complete. Popular uses of Ethereum have included the creation of fungible (ERC20) and non-fungible (ERC721) tokens with a variety of properties, crowdfunding (e.g. initial coin offerings), decentralized finance, decentralized exchanges, decentralized autonomous organizations (DAOs), games, prediction markets, and gambling. - -## How about scale for blockchain ? - -Concerning pure computing power, distributed computing offers easier scalability than centralized computing. It's relatively easy to add more machines to gain more computing power and reduce them when power needs are lower. - -However, blockchain has different scalability issues. In a blockchain, the number of transactions processed in a fixed period limits transaction speed. Therefore, the scalability issue is one of transaction speed. This scalability limitation is due to the need for the nodes in a blockchain to reach consensus on the transactions taking place. Therefore, while distributed computing itself offers a high degree of scalability, the game theory element of blockchain is generally what hampers scalability on transaction speeds. - -This gives rise to a concept of difficulty, Once there are too many computers involved in processing the transaction. The network will change the difficulty to reduce contention. The computer will have to calculate with higher difficulty. - -The downside is that performance issues arise because every node calculates all the smart contracts in real-time. As of January 2016, the Ethereum protocol could process about 25 transactions per second. In comparison, the Visa payment platform processes 45,000 payments per second. The next Ethereum 2.0 can serve more than 100.000 transactions - -Today we have a series of solutions to improve transaction speed on ethereum such as: Layer 2. - -## Reference - -https://en.wikipedia.org/wiki/Ethereum https://www.youtube.com/user/cbcolohan https://www.worldbank.org/en/topic/financialsector/brief/blockchain-dlt https://en.wikipedia.org/wiki/Distributed_ledger https://www.youtube.com/playlist?list=PLrw6a1wE39_tb2fErI4-WkMbsvGQk9_UB diff --git "a/topics/blockchain/\302\266 PoS.md" "b/topics/blockchain/\302\266 PoS.md" deleted file mode 100644 index 17740b50c..000000000 --- "a/topics/blockchain/\302\266 PoS.md" +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: PoS -description: This article provides an overview of Proof of Stake (PoS) in blockchain technology, including its definition, how it works, and its advantages. -date: 2022-06-16 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - foundational-topics - - po-s -redirect: - - /CHTrmw ---- - -## The proof of stake - -The proof of stake consensus algorithm was introduced in 2011 on the Bitcointalk forum to solve the problems of the most popular algorithm in use – Proof of Work. . While both share the same goal of achieving consensus in the blockchain, the process to achieve the goal is quite different. where the nodes of a blockchain-based network must "stake" a sum of money or tokens (proving their identity) in order to participate in the verification of transactions in a block. - -Just like proof of work, proof of stake is designed to achieve distributed consensus on the valid order of transactions - i.e. reach agreement on a single shared version of history . - -PoS indicates that a person can mine or confirm block transactions according to the amount of coins he or she holds. This means that the more Bitcoins or tokens owned by a miner, the more mining power will be available. - -The first cryptocurrency to adopt the PoS method was Peercoin. Nxt, Blackcoin and then ShadowCoin. - -![](assets/pos_jouvtnm.png.webp) - -## How proof of stake works - -The proof of stake algorithm uses a pseudo-random election process to select a node as the validator of the next block, based on a combination of factors that may include staking age, random and the size of the button. Users who wish to participate in this process must lock a certain number of native tokens into the network as their stake. The size of the stake determines the chance for a node to be selected as a validator to generate the next block – the larger the stake, the greater the chance. In order for the process not only to prioritize the wealthiest nodes in the network, unique methods are added to the selection process. The two most commonly used methods are 'Random Block Selection' and 'Coin Age Selection'. In the Random Block Pick method, validators are selected by looking for nodes with the combination of the lowest hash value and the highest stake, and since the stake size is public, the validator node The next real thing can usually be predicted by other nodes. The Coin Age Selection method selects nodes based on how long their coins have been staked. Coin age is calculated by multiplying the number of days the coin is held by the number of coins staked. When a node forges a block, their coin age is reset to zero and they have to wait a certain amount of time before they can generate another block – this prevents large stake nodes from dominating the blockchain. - -Each native token that uses the Proof of Stake algorithm has its own set of rules and methods to create the best combination for them and their users. - -When a node is selected to generate the next block, it checks if the transactions in the block are valid, signs the block, and adds it to the blockchain. The node receives the transaction fees associated with the transactions in the block. - -If a node wants to stop working as a blacksmith, its staked coins along with the rewards earned are released after a certain amount of time, giving the network time to verify that no fraudulent blocks were added by that node. into the blockchain. - -Proof of stake was created as an alternative to proof of work (PoW), to solve the inherent problems of computation time and energy consumption when using PoW. - -PoS seeks to solve the problem by reducing mining power to the percentage of coins a miner spends to join the nodes. This way, instead of using energy to solve the PoW problem, PoS miners are limited to mining by a transaction rate that reflects the number of shares the miner owns. For example, a miner who owns 3% of Bitcoins could theoretically only mine 3% of those blocks. - -#### Compare PoW and PoS - -![](assets/pos_xiuwh4m.png.webp) - -## About security - -The stake coin acts as a financial incentive for the forging node to not validate or generate fraudulent transactions. If the network detects a fraudulent transaction, the forging node will lose part of its stake and the right to participate in future block forging. So as long as the stake is higher than the reward, validators will lose more coins than they would have gained in case of a fraud attempt. - -To effectively control the network and approve fraudulent transactions, a node would have to own a majority stake in the network, this is known as a 51% attack. Depending on the value of the native token, this would be very impractical as to gain control of the network you would need to have more than 51% of the circulating supply. - -In 2017, Ethereum (ETH) started its full transition from PoW to PoS system and by 2022 it has successfully deployed on Ropsten testnet. - -## Advantages of PoS: - -- Fast transaction processing. -- PoS does not harm the environment. -- Not vulnerable to government attacks: don't need huge amounts of electricity. -- Can be performed on smaller and weaker devices because there is no need to download the entire blockchain, and because it does not require a lot of computing power, it can be diff --git "a/topics/blockchain/\302\266 Smart Contract.md" "b/topics/blockchain/\302\266 Smart Contract.md" deleted file mode 100644 index 0594dc168..000000000 --- "a/topics/blockchain/\302\266 Smart Contract.md" +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Smart contract -description: This article provides an overview of smart contracts in blockchain technology, including their definition, how they work, and their advantages. -date: 2022-06-16 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - foundational-topics - - smart-contract -redirect: - - /GS5dBg ---- - -## Smart contracts - -Smart contracts, also known as smart contracts, are computer programs that operate on the blockchain. Entities interacting with the contract do not need to know each other or trust each other. - -- The contract ensures that with the conditions of the contract satisfied, the contract will be executed -- The logic of the contract can be verified. In other words, a smart contract is an intermediary, greatly reducing operating costs if you do it in real life: for example, if you buy a house, you must notarize it, then pay. money at the notary office, then go to the real estate office to do the account transfer procedure... it will need many steps involving people and intermediaries. If you make this purchase on a smart contract, you will simply create a home sale transaction and one buyer will create a purchase transaction, the other contract will perform the transfer of money, change ownership, related information, and sales history. - -## How smart contract work: - -- Developer distributes contract on the blockchain -- The user signs the transaction and invokes the contract on the blockchain -- Contract processes data and executes commands -- The data after being executed will be saved on the blockchain - -## Structure of a smart contract: - -- Declare solidity version -- Declare libraries, interfaces -- Declare variables -- Declare constructor -- Processing instructions -- Save state -- Event/Log - -Declare variable - -Function/Instruction - -Event/Log - -## Invoke another smart contract: - -Currently, Ethereum there are many contracts such as tokens, NFT-ERC721, games, swaps, lending... To work directly with these contracts you can make calls directly from your application from javascript or go lang via ABI, ABI is an interface type similar to API specification file or description file. describe swagger. It defines data objects and callable functions. Or you can also call the interface of another contract in your contract for example: - -- Declare interface -- Call command - -## The advantages of smart contracts: - -- Efficiency: Smart contracts promise to automate business processes at a corporate level. This reduces operating costs and saves resources, including the staff needed to oversee complex operations involving multiple companies. -- Processing speed: Smart contracts help improve the processing speed of processes between many different companies and corporations. -- Autonomy: Smart contracts are executed automatically by a network and help reduce the need for a 3rd party to manage transactions between companies. -- Reliability: Smart contracts also leverage blockchain ledger and other distributed ledger technologies to store all the information and operations involved in complex processing after it has been executed presently. This technology also supports automated trading which eliminates human errors and ensures accuracy in contract execution. - -## The limitations of smart contracts: - -- Security issues: Smart contracts play certain important roles in a business involving many parties. However, this technology is still new and hackers are constantly exploiting new attack directions to penetrate. In the early days of Ethereum, hackers hacked and stole a large amount of virtual currency worth $50 million. The IEEE Consortium of Electrical and Electronic Engineers has also expressed concern about the weakness of the tools used to detect vulnerabilities in smart contracts. -- Integrity: An oracle (a data source that sends event updates) should be protected from hackers creating fake events to trigger the processing of contracts even though they are not allowed. The system needs to be programmed to generate the correct events, which can be quite difficult in complex cases. -- Relevancy: Smart contracts can speed up processing in a multi-party contract regardless of whether it matches the intent or understanding of all parties. But it can also add to the damage in the event things get out of hand, especially when there's no way to stop or reverse the unintended actions. Research firm Gartner has pointed out that this creates a challenge in the management of smart contracts, although this challenge has not been fully addressed. -- Complexity in management: Smart contracts are quite complicated in deployment and management. They are often designed in such a way that it is very difficult or impossible to change. Although this increases security, the parties will not be able to change the content or add new terms without creating a new contract. - -Take for example a smart contract when deployed When a Decentralized Autonomous Organization (DAO) named "The DAO" was hacked in 2016, millions of ETH were stolen due to a mistake in their smart contract code. Since their Smart contract is immutable, developers cannot edit the code. This eventually led to a hard fork, creating Ethereum Classic and Ethereum. - -#### Reference - -https://ethereum.org/vi/developers/docs/smart-contracts/ diff --git "a/topics/blockchain/\302\266 Zero-knowledge Proofs.md" "b/topics/blockchain/\302\266 Zero-knowledge Proofs.md" deleted file mode 100644 index ec170ac5f..000000000 --- "a/topics/blockchain/\302\266 Zero-knowledge Proofs.md" +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: Zero-knowledge proofs -description: This article provides an overview of zero-knowledge proofs in blockchain technology, including their definition, how they work, and their advantages. -date: 2022-09-06 -authors: - - thanhpn -github_id: thanhpn -tags: - - blockchain - - evm - - zk-proof -redirect: - - /ILCGXw ---- - -Zero-knowledge proof is a way of proving the validity of a statement without disclosing the statement itself. A "validator" is the party attempting to prove a claim, while a "verifier" is responsible for validating the claim. - -## Why do we need to demonstrate no knowledge? - -Zero-knowledge proofs represent a breakthrough in applied cryptography, as they promise to improve the security of information for individuals. Consider how you can prove the claim (for example, “I am a citizen of country X”) to another party (for example, a service provider). You'll need to provide "proof" to back up your claim, such as a national passport or driver's license. But this approach is not safe, can be hacked, personal information can be revealed - -Zero-knowledge proofs solve this problem by removing the need to disclose information to prove the validity of claims. The zero-knowledge protocol uses a statement (called a 'witness') as input to generate a succint proof of its validity. This proof provides firm assurance that a statement is true without revealing the information used to make it. - -Going back to our earlier example, the only proof you need to prove your citizenship claim is zero-knowledge proof. The verifier only has to check if certain properties of the proof are true to believe that the underlying statement is also true. - -## How to prove Zero Knowledge works? - -To make this possible, zero-knowledge protocols rely on algorithms that take some data as input and return the 'true' or 'false' as output. A Zero-knowledge protocol must satisfy the following criteria: - -1. **Completeness**: If the input is valid, the zero-knowledge protocol always returns 'true'. Hence, if the underlying statement is true, and the prover and verifier act honestly, the proof can be accepted. -2. **Soundness**: If the input is invalid, it is theoretically impossible to fool the zero-knowledge protocol to return 'true'. Hence, a lying prover cannot trick an honest verifier into believing an invalid statement is valid (except with a tiny margin of probability). -3. **Zero-knowledge**: The verifier learns nothing about a statement beyond its validity or falsity (they have “zero knowledge” of the statement). This requirement also prevents the verifier from deriving the original input (the statement’s contents) from the proof. - -In basic form, a zero-knowledge proof is made up of three elements: witness, challenge, and response. - -- **Witness**: with a zero-knowledge proof, the prover wants to prove knowledge of some hidden information. the secret information is the "witness" to the proof, and the prover's assumed knowledge of the witness establishes a set of questions that can only be answered by a party with knowledge of the information. Thus, the prover starts the proving process by randomly choosing a question, calculating the answer, and sending it to the verifier. -- **Challenge**: The verifier randomly picks another question from the set and ask the prover to answer it. -- **Response**: the prover accepts the question, calculates the answer and returns it to the verifier. The prover's response allows the verifier to check if the former really has access to the witness. to ensure the prover is not guessing blindly and getting the correct answers by chance, the verifier pick more question to ask. By repeating this interaction many times, the possibility of the prover faking knowledge of the witness drops significant until the verifier is satisfied. - -Interactive proof and non-interactive proof: - -- Interactive proof had limited usefulness since it required the two parties to be available and interact repeatedly -- Non-interactive proof required only one round of communication between participants. the provers passes the secret information to a special algorithm to compute a zero-knowledge proof. this proof is sent to the verifier, who verify that the prover knows the secret information using other algorithm. - -## Types of zero-knowledge proofs - -### ZK-SNARKs - -ZK-SNARK is an acronym for Zero-Knowledge Succinct Non-Interactive Argument of Knowledge. The ZK-SNARK protocol has the following qualities: - -- **Zero-knowledge**: A verifier can validate the integrity of a statement without knowing anything else about the statement. The only knowledge the verifier has of the statement is whether it is true or false. -- **Succinct**: The zero-knowledge proof is smaller than the witness and can be verified quickly. -- **Non-interactive**: The proof is 'non-interactive' because the prover and verifier only interact once, unlike interactive proofs that require multiple rounds of communication. -- **Witness**: The proof satisfies the 'soundness' requirement, so cheating is extremely unlikely. -- **(Of) Knowledge**: The zero-knowledge proof cannot be constructed without access to the secret information (witness). It is difficult, if not impossible, for a prover who doesn’t have the witness to compute a valid zero-knowledge proof. - -For the ZK-SNARK protocol to work, the creation of a Common Reference String (CRS) is necessary: ​​The CRS provides public parameters to prove and verify valid proofs. The security of the proof system depends on the CRS setting; If the information used to create the public parameters falls into the possession of malicious actors, they can create false validators. - -- Some ZK-rollups attempt to solve this problem by using multiparty computation (MPC), involving trusted individuals, to create public parameters for the ZK-SNARK circuit. Each party contributes a random number (called "hazardous waste") to the construction of the CRS, which they must destroy immediately. -- Trusted settings are used because they increase the security of the CRS setup. As long as an honest participant discards their input, the security of the ZK-SNARK system is guaranteed. However, this approach still requires the trust of the stakeholders to erase their sampled randomness and not undermine the security guarantees of the system. -- Reliability assumptions aside, ZK-SNARK is very popular because of its small proof size and continuous time verification. Since verifying proofs on L1 constitutes a greater operating cost of ZK-rollup, L2 uses ZK-SNARK to generate proofs that can be quickly and cheaply verified on the Mainnet. - -### ZK-STARKs - -Like ZK-SNARKs, ZK-STARKs demonstrate the validity of off-chain computation without revealing the input. However, ZK-STARK is considered an improvement on ZK-SNARK because of their scalability and transparency. - -- **Scalable**: ZK-STARK is faster than ZK-SNARK in generating and verifying evidence when witness size is larger. With STARK proofs, verification and proverb times only increase slightly as the witness grows (the times of the proverb and SNARK verifier increase linearly with witness size). -- **Transparency**: ZK-STARK relies on public verifiable randomness to generate public parameters for proof and verification instead of establishing trust. Therefore, they are more transparent than ZK-SNARK. -- **Scalability**:ZK-STARKs also offer more scalability because the time required to prove and verify valid proofs increases with the complexity of the underlying computation. With ZK-SNARK, the proof and verification times expand linearly with respect to the size of the underlying computation. This means that ZK-STARK requires less time than ZK-SNARK to prove and verify as far as large data sets are concerned, making them useful for high volume applications. -- **Security**: ZK-STARK is also secure against quantum computers, while Elliptic Curve Cryptography (ECC) used in ZK-SNARK is considered by many to be vulnerable to quantum computing attacks. The downside of ZK-STARKs is that they produce a larger proof size, which is more expensive to verify on Ethereum. Also, they don't support recursion, which is key to extending off-chain computation with zero-knowledge proofs. - -## Application for ZK proof - -- Anonymous payments -- Identity protection -- Authentication -- Verifiable computation - -## Drawbacks of using ZK proofs - -- Hardware costs -- Proof verification costs -- Trust assumptions -- Quantum computing threats - -## References - -- [Zero-knowledge proofs](https://ethereum.org/en/zero-knowledge-proofs/) -- [ZL-Rollup](https://docs.ethhub.io/ethereum-roadmap/layer-2-scaling/zk-rollups/) -- [ZK-SNARKs](https://medium.com/coinmonks/zk-snarks-a-realistic-zero-knowledge-example-and-deep-dive-c5e6eaa7131c) -- [ZK-STARKs](https://medium.com/coinmonks/zk-starks-create-verifiable-trust-even-against-quantum-computers-dd9c6a2bb13d) -- [Snarks-vs-starks](https://www.alchemy.com/overviews/snarks-vs-starks) diff --git a/topics/communication/Economic narratives spread like diseases.md b/topics/communication/Economic narratives spread like diseases.md deleted file mode 100644 index 68b6e4c03..000000000 --- a/topics/communication/Economic narratives spread like diseases.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - narrative - - story-telling -redirect: - - /Ryljag ---- - -The spread of economic narratives follows a similar pattern to the spread of diseases: there is a rapid rise in the number of the “infected” people spreading the story, but over time, the curve flattens and begins to diminish as people forget and lose interest in talking about the narrative. Spread follows a familiar hump-shaped pattern of rising then falling over months, years, or even decades. - -Nevertheless, it may rise again in the future. A narrative is nearly impossible to eradicate. Details may mutate, but often the underlying fundamentals persist. For instance, narratives surrounding bitcoin hearken back to economic narratives about bimetallism in the nineteenth century. Similarly, contemporary narratives about machine learning and automation share much with Luddite narratives as well as narratives about “cybernation” in the 1960s. - -Economic narratives are difficult to trace and predict. The economy is complex, and so are the many conflicting narratives and ideas that are relevant to economic decision-making. Like a contagion, an economic narrative may recur in different places and times. The spread of a narrative has little to do with its quality, importance, or accuracy and much more to do with seemingly random details, such as the frequency with which it is encountered. Much like diseases, narratives may be spread more rapidly by super-spreaders including the media as well as algorithm-driven marketing or social media channels. - ---- - -#### Related - -- [[Strength of weak ties]] -- [[Ideas are networks]] -- [[Innovation thrives in open environments]] -- [[Metaphors structure our interpretation of the world]] -- [[≈ Narrative Economics]] - -#### Reference - -[[≈ Narrative Economics | Shiller, Robert J. Narrative Economics: How Stories Go Viral and Drive Major Economic Events. Princeton: Princeton University Press, 2019.]] diff --git a/topics/communication/Metamodernism.md b/topics/communication/Metamodernism.md deleted file mode 100644 index 7434431d5..000000000 --- a/topics/communication/Metamodernism.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - story-telling -redirect: - - /ShlBfQ ---- - -Metamodernism, a cultural mode following modernism and postmodernism, describes a pattern of cultural philosophy emerged in the late twentieth century. The term was coined in 1975 by Mas'ud Zavarzade, but has become associated primarily with the Age of the Internet. - -According to theorists Timotheus Vermeulen and Robin van den Akker, metamodernism is "a kind of informed naivety a pragmatic idealism." It embraces the grand narratives of modernism, but nevertheless understands them as being problematic. Fabio Vittorini notes that in metamodernism, we see the collision of "naive and/or fanatic idealism" with "skeptical and/or apathetic pragmatism," suggestive that metamodernism is, if nothing else, fraught with contradictions. Metamodernism is characterized by genuine connection, empathy, and community; but also by ambiguity, reconstruction, collaboration, and paradox. - -Metamodernism works often combine honesty with irony, and "remix" past cultural materials in new combinations. However, in contrast to postmodern pastiche, these works are usually produced with sincerity rather than cynicism. - ---- - -#### Reference - -Cunff, Anne-Laure Le. “An Introduction to Metamodernism: The Cultural Philosophy of the Digital Age.” _Ness Labs_ (blog), December 12, 2019. [https://nesslabs.com/metamodernism](https://nesslabs.com/metamodernism). diff --git a/topics/communication/Metaphors create coherent systems of meaning.md b/topics/communication/Metaphors create coherent systems of meaning.md deleted file mode 100644 index 843f90700..000000000 --- a/topics/communication/Metaphors create coherent systems of meaning.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - metaphor - - story-telling -redirect: - - /Gvi99A ---- - -"Metaphor" means understanding one thing in terms of another. This can be explicit or subtle. Sometimes—often?—metaphors are such a deep part of our psyche that we're unaware that we are using a metaphor at all. For instance, describing an argument in terms of war, or an idea in terms of a structure or building. Metaphors form coherent systems: when a concept is described using different metaphors, these usually fit together in a logical way. - ---- - -#### Related - -- [[Metaphors structure our interpretation of the world]] -- [[Narratives generate real-world effects]] -- [[≈ Metaphors We Live By]] -- [[§ Writing]] - -#### Reference - -[[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/Metonymy facilitates understanding by focusing our attention.md b/topics/communication/Metonymy facilitates understanding by focusing our attention.md deleted file mode 100644 index 15ac63347..000000000 --- a/topics/communication/Metonymy facilitates understanding by focusing our attention.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - metaphor -redirect: - - /UcsYww ---- - -Metaphor and metonymy are each distinct processes. Where metaphor describes one thing in terms of another, metonymy allows one thing to stand in for another. Metonmy also facilitates understanding, while allowing us to focus on specific aspects of what's being referred to. - ---- - -#### Related - -- [[Metaphors structure our interpretation of the world]] -- [[Metaphors create coherent systems of meaning]] -- [[Structural metaphors reveal and conceal different aspects of experience]] -- [[Orientational metaphors organize concepts relationally]] - -#### Reference - -[[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/Narratives generate real-world effects.md b/topics/communication/Narratives generate real-world effects.md deleted file mode 100644 index 148f8fb48..000000000 --- a/topics/communication/Narratives generate real-world effects.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - narrative - - story-telling -redirect: - - /7yT2ywQ ---- - -Narratives, alone or in constellation with one another, produce real-world effects. People use narratives to help them interpret events. As [Kahnemann](https://publish.obsidian.md/mobydiction/Kahnemann) and Tversky outline in their model of the "representativeness heuristic," stories frame people's expectations and therefore inform how they will behave. - -Robert Shiller describes how [[¶ Economic narrative | narratives]] inform individual economic actions, such as when and where to invest or whether to take a certain job. But, a story that is virulent enough can have mass-scale impacts. For instance, narratives in the latter half of the twentieth century that used efficiency to describe free markets helped contribute to a public reaction against regulation and government intervention in the economy. - ---- - -#### Reference - -- [[Narrative sequencing informs meaning]] -- [[Narrative helps the human mind make sense of events]] -- [[Narratives enable us to act decisively in conditions of uncertainty]] -- [[≈ Narrative Economics]] diff --git a/topics/communication/Novel metaphors create novel ways of thinking.md b/topics/communication/Novel metaphors create novel ways of thinking.md deleted file mode 100644 index ab565f35b..000000000 --- a/topics/communication/Novel metaphors create novel ways of thinking.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - metaphor - - story-telling -redirect: - - /ATbuvg ---- - -Imaginative or novel metaphors offer new ways of thinking about familiar ideas or concepts. They either extend existing metaphors, develop previously unused parts of an existing metaphor, or provide a completely new metaphor and, therefore, way of looking at something. In other words, "new metaphors have the power to create a new reality" insofar as they provide new ways to interpret to interpret, and then respond to, experiences. They may further help us create new similarities between previously unlike ideas, concepts, or objects. Metaphorical shifts, therefore, can even inspire cultural and social change. - ---- - -#### Related - -- [[Metaphors structure our interpretation of the world]] -- [[Metaphors create coherent systems of meaning]] -- [[Narratives generate real-world effects]] -- [[Structural metaphors reveal and conceal different aspects of experience]] -- [[Orientational metaphors organize concepts relationally]] -- [[Metonymy facilitates understanding by focusing our attention]] -- [[≈ Metaphors We Live By]] - -#### Reference - -[[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/Objectivism assumes a concrete, knowable world.md b/topics/communication/Objectivism assumes a concrete, knowable world.md deleted file mode 100644 index ce092f21f..000000000 --- a/topics/communication/Objectivism assumes a concrete, knowable world.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /Z1VrFA ---- - -"Objectivism" assumes that the world comprises concrete, knowable objects that are classified in terms of inherent properties. According to objectivism, there is an objective reality; but, because humans are prone to making errors, we cannot trust their subjective judgment. Therefore, we turn to science to provide us with a methodology for evaluating what is true. Objectivity is a fundamental good, and only objective knowledge is true knowledge. Objectivism has come to be the predominant perspective in Western society today. It offers the comfort of consistency and clarity. - ---- - -#### Related - -- [[Four Theories of Truth]] -- [[Rationalistic models of human behaviour are limiting]] -- [[≈ Metaphors We Live By]] - -#### Reference - -[[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/Orientational metaphors organize concepts relationally.md b/topics/communication/Orientational metaphors organize concepts relationally.md deleted file mode 100644 index c0057d6fd..000000000 --- a/topics/communication/Orientational metaphors organize concepts relationally.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /bd1Wug ---- - -Orientational metaphors organize concepts in relation to one another, typically using spatial orientations. Orientational metaphors are especially useful for conceptualizing emotions, which typically lack a clearly defined conceptual structure in and of themselves. For instance, "happy" is up while "sad" is down. These are structured in large part from deep-seated cultural values. - ---- - -#### Related - -- [[Metaphors structure our interpretation of the world]] -- [[Metaphors create coherent systems of meaning]] -- [[Structural metaphors reveal and conceal different aspects of experience]] -- [[≈ Metaphors We Live By]] - -#### Reference - -- [[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/Perspective comprises our stance, our tools, and our experience.md b/topics/communication/Perspective comprises our stance, our tools, and our experience.md deleted file mode 100644 index 6426e465c..000000000 --- a/topics/communication/Perspective comprises our stance, our tools, and our experience.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /PKTgAA ---- - -According to Roger Martin, perspective on a given issue is formed by the interaction between three things: our **stance**, our **tools**, and our **experience**. - -**Stance** describes our personal domain of knowledge as well as our sense of ourselves and our purpose. This frames our personal perspective and approach to the problem space. - -**Tools** includes not just physical tools but also the thinking tools we bring to bear on a situation: theories, processes, rules of thumbs, best practices, and so on. We use these to organize complex problem spaces into something more accessible. - -**Experience** helps us hone our intuition and informs our sensitivities when regarding a problem or challenge. - -These different elements do not flow linearly from one to another; they mutually inform each other. - ---- - -#### Related - -- [Honing intuition](https://publish.obsidian.md/mobydiction/notes/Honing+intuition) \- We can hone our intuition by acquiring new knowledge and experiences. -- [Salience is subjective](https://publish.obsidian.md/mobydiction/notes/Salience+is+subjective) \- What we view as salient to a problem or situation is informed by our subjective experiences -- [[≈ The Opposable Mind]] - -#### Reference - -Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007. diff --git a/topics/communication/Sensemaking produces thick data.md b/topics/communication/Sensemaking produces thick data.md deleted file mode 100644 index 8a678b358..000000000 --- a/topics/communication/Sensemaking produces thick data.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /59fWwg ---- - -[[¶ Sensemaking | Sensemaking]], a qualitative, humanities-based approach to developing understanding of a culture, produces what Christian Madsbjerg calls "thick data." In contrast to the [[Quant data is lossy | thin, broad data]] produced by more scientific approaches, thick data goes below the surface to find the rich context behind facts. Beyond observing that an event occurred, thick data includes information on what the experience of the event feels like or means to a culture or population, and how they relate to it. - -Thick data is often overlooked because it is less universally applicable than thin data. But without it, we cannot claim to have a complete model of humanity. When we are attuned to thick data, we are more sensitive to subtle fluctuations in the environments we observe and inhabit; our intuition is stronger; and we are better equipped to make decisions in the "wicked" environment of human relations—including business. - ---- - -#### Related - -- [[Qualitative research creates mental models about the problem space]] -- [[Quantitative analysis is not inherently more reliable than qualitative data]] -- [Quant data is lossy](https://publish.obsidian.md/mobydiction/notes/Quant+data+is+lossy) -- [[¶ Sensemaking]] - -#### Reference - -Madsbjerg, Christian. Sensemaking: The Power of the Humanities in the Age of the Algorithm. Hachette Books, 2017. diff --git a/topics/communication/Structural metaphors reveal and conceal different aspects of experience.md b/topics/communication/Structural metaphors reveal and conceal different aspects of experience.md deleted file mode 100644 index 998ed5a18..000000000 --- a/topics/communication/Structural metaphors reveal and conceal different aspects of experience.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - metaphor -redirect: - - /lQgKNw ---- - -Structural metaphors emphasize some aspects of experience but hide others. For example, using martial terms to discuss argument occludes argument as a potentially cooperative activity. Metaphors can never capture the full aspect of what they are used to describe; therefore, they shape our perspective of what they describe. They reveal and conceal. In so doing, they may sanction or prohibit different inferences and actions by implying an appropriate relation to what is being described. - -Some examples of structural metaphors: - -- Arguments are battles -- Ideas are objects -- Expressions are containers -- Communication is sending - ---- - -#### Related - -- [[Metaphors create coherent systems of meaning]] -- [[Metaphors structure our interpretation of the world]] -- [[≈ Metaphors We Live By]] - -#### Reference - -[[≈ Metaphors We Live By | Lakoff, George, and Mark Johnson. Metaphors We Live By. University of Chicago Press, 2008.]] diff --git a/topics/communication/cognitive-diversity-enhances-team-problem-solving-capabilities.md b/topics/communication/cognitive-diversity-enhances-team-problem-solving-capabilities.md deleted file mode 100644 index 7620412dc..000000000 --- a/topics/communication/cognitive-diversity-enhances-team-problem-solving-capabilities.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Cognitive diversity enhances team problem solving capabilities -description: Teams with cognitive diversity solve problems faster and think more creatively than homogeneous groups, showing the value of diverse thinking styles in improving team performance and problem-solving. -date: null -redirect: - - /IKPLDw ---- - -A study conducted by Alison Reynolds and David Lewis found that teams that comprised individuals representing a variety of thinking or information processing styles (i.e. cognitive diversity) solved problems faster than cognitively homogeneous groups. This suggests that organizations or teams that hire for "fit" may be hampering their own problem-solving capabilities and creative acuity. - -Roger L. Martin suggests that organizing teams for diverse specializations can also broaden their sense of salience when considering a problem; this affords a stronger, more holistic view of the problem or situation. - ---- - -#### Related - -- [[Specialization narrows our perspective on what is salient to a problem]] - Specialization offers mastery of some mental models, but filters our view of the problem. -- [[¶ Consilience]] - A concept describing the synthesis of knowledge produced from disparate fields - -#### Citation - -Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007. - -Reynolds, Alison, and David Lewis. “Teams Solve Problems Faster When They’re More Cognitively Diverse.” _Harvard Business Review_, March 30, 2017. [Link](https://hbr.org/2017/03/teams-solve-problems-faster-when-theyre-more-cognitively-diverse). diff --git a/topics/communication/expressing-uncertainty-doesn-t-undermine-authority.md b/topics/communication/expressing-uncertainty-doesn-t-undermine-authority.md deleted file mode 100644 index ef469e52d..000000000 --- a/topics/communication/expressing-uncertainty-doesn-t-undermine-authority.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Expressing uncertainty doesn t undermine authority -description: Researchers should openly share uncertainties in their findings, as expressing doubt builds trust and helps audiences better evaluate evidence, especially in changing situations like COVID-19. -date: null -redirect: - - /U5kZ0A ---- - -When reporting their findings, researchers should not be reluctant to admitting what they don't know. What we know might remain in flux as new evidence comes to light, and identifying remaining uncertainties is more informative than presenting false confidence in an assertion. Indeed, uncertainty may be a critical part of the message that needs to be shared, such as when reporting on dynamic situations (like COVID guidelines and trends). - -Blastland et al. found that expressing uncertainty did not have a negative impact on the trustworthiness of reporting. They further add that there's little downside in expressing findings in a range rather than as an absolute number. Blastland et al. suggest that audiences are more sophisticated at evaluating arguments and evidence than popular media tend to give them credit for and that they will, in fact, evaluate claims based on strength of evidence more than clarity of message. - ---- - -#### Related - -- [[Inform rather than persuade]] - -#### Citations - -Blastland, Michael, Alexandra L. J. Freeman, Sander van der Linden, Theresa M. Marteau, and David Spiegelhalter. _“Five Rules for Evidence Communication.”_ Nature 587, no. 7834 (November 2020): 362–64. [Link](https://doi.org/10.1038/d41586-020-03189-1). diff --git a/topics/communication/inform-rather-than-persuade.md b/topics/communication/inform-rather-than-persuade.md deleted file mode 100644 index 88eb082aa..000000000 --- a/topics/communication/inform-rather-than-persuade.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Inform rather than persuade -description: Learn how evidence communication promotes honest, transparent sharing of research by presenting uncertainties and conflicts to inform audiences without persuasion or bias. -date: null -redirect: - - /cs2sgg ---- - -Researchers sharing results should strive to inform their audiences rather than persuade them with partisan reporting. The goal should not be to change beliefs, but to provide people with information that they can determine themselves how best to respond to. This means avoiding neat, tidy narratives and focusing instead of honesty and transparency. - -This approach, dubbed "evidence communication" by Blastland et al., is characterized by transparency around the motivation of the researchers, a frank depiction of uncertainties, and open identification of conflicts and contradictions. The aim is to "inform rather than persuade" and to promote trust in the researchers and their work. - -Misunderstandings and misconceptions should be anticipated and "pre-bunked," and findings should be presented in totality rather than cherry-picked to best support an argument. Pros and cons should be identified and presented side-by-side to enable the audience to make the best choice. Sources should be clearly identified, and the strength (or weakness) of the evidence delivered with "unapologetic uncertainty." - ---- - -#### Related - -- [[Expressing uncertainty doesn't undermine authority]] - Blastland et al. found that identifying uncertainties did not undermine audience trust in the research -- [[Narrative helps the human mind make sense of events]] - Narratives simplify, but in so doing help us comprehend complexity -- [[Narratives enable us to act decisively in conditions of uncertainty]] - When driving action, narratives may help promote decisiveness -- [[Narrative sequencing informs meaning]] - The arrangement of insights informs how we interpret them; our aims—persuasion or information—inform that arrangement. - -#### Citations - -Blastland, Michael, Alexandra L. J. Freeman, Sander van der Linden, Theresa M. Marteau, and David Spiegelhalter. _“Five Rules for Evidence Communication.”_ Nature 587, no. 7834 (November 2020): 362–64. [Link](https://doi.org/10.1038/d41586-020-03189-1). diff --git a/topics/communication/perspective-comprises-our-stance-our-tools-and-our-experience.md b/topics/communication/perspective-comprises-our-stance-our-tools-and-our-experience.md deleted file mode 100644 index 5a5be8131..000000000 --- a/topics/communication/perspective-comprises-our-stance-our-tools-and-our-experience.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Perspective comprises our stance our tools and our experience -description: Learn how perspective is shaped by the interaction of stance, tools, and experience to better understand and approach complex problems effectively. -date: null -redirect: - - /zR3teQ ---- - -According to Roger Martin, perspective on a given issue is formed by the interaction between three things: our **stance**, our **tools**, and our **experience**. - -**Stance** describes our personal domain of knowledge as well as our sense of ourselves and our purpose. This frames our personal perspective and approach to the problem space. - -**Tools** includes not just physical tools but also the thinking tools we bring to bear on a situation: theories, processes, rules of thumbs, best practices, and so on. We use these to organize complex problem spaces into something more accessible. - -**Experience** helps us hone our intuition and informs our sensitivities when regarding a problem or challenge. - -These different elements do not flow linearly from one to another; they mutually inform each other. - ---- - -#### Related - -- [Honing intuition](https://publish.obsidian.md/mobydiction/notes/Honing+intuition) \- We can hone our intuition by acquiring new knowledge and experiences. -- [Salience is subjective](https://publish.obsidian.md/mobydiction/notes/Salience+is+subjective) \- What we view as salient to a problem or situation is informed by our subjective experiences - -#### Citations - -Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007. diff --git a/topics/communication/put-mental-models-in-conversation-with-one-another.md b/topics/communication/put-mental-models-in-conversation-with-one-another.md deleted file mode 100644 index f8b4ae42e..000000000 --- a/topics/communication/put-mental-models-in-conversation-with-one-another.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Put mental models in conversation with one another -description: Learn how applying multiple mental models and diverse perspectives improves decision-making by revealing hidden insights and overcoming narrow, specialized thinking. -date: null -redirect: - - /P6lnow ---- - -Mental models are tools that help us more efficiently arrive at decisions: they filter information and give it narrative shape. But they always represent a particular lens. They help make meaning, but meaning comes at the expense of information. - -Relying on only one mental model limits our view of the problem space. To overcome this limitation, try to apply multiple mental models to the situation. Select models that are potentially antagonistic with one another and use them to find salience or causal relationships among the data that cannot be anticipated by any model alone. - -This can be challenging to accomplish, as we tend to specialize over time, returning to comfortable, reliable tools of thought. Bring in collaborators who can apply different perspectives to help overcome this potential gap. - -Roger Martin calls this practice "assertive inquiry." - ---- - -#### Related - -- [[Narratives enable us to act decisively in conditions of uncertainty]] - A mental model is a kind of narrative to help us maneuver through complexity. -- [Cognitive diversity enhances team problem solving capabilities](https://publish.obsidian.md/mobydiction/notes/Cognitive+diversity+enhances+team+problem+solving+capabilities) \- Teams that are cognitively diverse expend our view of what is salient to the problem space. -- [Specialization narrows our perspective on what is salient to a problem](https://publish.obsidian.md/mobydiction/notes/Specialization+narrows+our+perspective+on+what+is+salient+to+a+problem) \- When we specialize, we risk narrowing our focus and might overlook salient information or data. -- [Protect dissent](https://publish.obsidian.md/mobydiction/notes/Protect+dissent) \- It's important to enshrine the capacity to question decisions and push back against popular modes of thought. - -#### Citations - -[Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007.](https://publish.obsidian.md/mobydiction/notes/%E2%89%88+Martin+-+The+Opposable+Mind) diff --git a/topics/communication/salience-is-subjective.md b/topics/communication/salience-is-subjective.md deleted file mode 100644 index 685b9b374..000000000 --- a/topics/communication/salience-is-subjective.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Salience is subjective -description: Learn how Roger Martin explains problem solving by focusing on the most important information shaped by our stance, tools, and experience to improve decision-making and perspective. -date: null -redirect: - - /cii1GA ---- - -Problem solving, Roger Martin writes, involves first identifying the features of a problem that are most important. We can't possibly consider every facet of the challenge ahead, so we must pick and choose what we will care about. We therefore have to assess the salience of available information to the problem at hand. - -This sense of salience can vary dramatically between different people: it is informed, Martin says, by each individual's _stance_—our personal domain of knowledge, sense of self, and sense of purpose—their _tools_—including literal tools but also the theories, models, processes, and rules of thumb that we bring to the problem space—and their _experience_—the accrued practical, tangible knowledge that has honed their intuition and sensitivity to different elements of the problem. - ---- - -#### Related - -- [[Cognitive diversity enhances team problem solving capabilities]] - Building a team that is cognitively diverse expends our sense of salience -- [[Specialization narrows our perspective on what is salient to a problem]] - Specialization frames what we view as salient -- [[Our vision of the future is always constrained by our view of the past]] - Our past experience and training constrains our vision -- [[Perspective comprises our stance, our tools, and our experience]] - Describes elements that form our perspective - -#### Citations - -Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007. diff --git "a/topics/communication/\302\266 Constellation.md" "b/topics/communication/\302\266 Constellation.md" deleted file mode 100644 index a64a37661..000000000 --- "a/topics/communication/\302\266 Constellation.md" +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication - - innovation -redirect: - - /l1OA8t0 ---- - -According to Robert Shiller, large-scale [[¶ Economic narrative | economic narratives]] may comprise many other, smaller narratives. Shiller calls these constellations: They are narratives that reinforce one another and are made more plausible by dint of their relationship to other narratives. No single economic event can be explained by just one economic narrative; they are not so easily explicable. Rather, they emerge in constellations. - ---- - -#### Related - -- [[Ideas are constellations]] -- [[Economic narratives spread like diseases]] -- [[Narrative helps the human mind make sense of events]] -- [[≈ Narrative Economics]] diff --git "a/topics/communication/\302\266 Economic narrative.md" "b/topics/communication/\302\266 Economic narrative.md" deleted file mode 100644 index 649ed043e..000000000 --- "a/topics/communication/\302\266 Economic narrative.md" +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - story-telling - - writing -redirect: - - /H0MR6w ---- - -An "economic narrative" is a contagious story that has the potential to change how people make economic decisions, such as how, why, and where to invest; how much to spend and to save; and whether to buy a home or take a certain job. - -An economic narrative, according to Robert Shiller, has the following characteristics: - -- It reminds people of facts they might have forgotten -- It offers an explanation about how things work in the economy -- It affects how people think about the justification or purpose of economic actions - -When these stories are virulent enough, they can have a significant impact on the economy. For instance, narratives in the latter half of the twentieth century that used efficiency to describe free markets helped contribute to a public reaction against regulation and government intervention in the economy. - -[[Economic narratives spread like diseases | Shiller says that these narratives spread in a manner similar to contagions]] - -Larger-scale narratives may comprise multiple smaller narratives. Shiller calls these “constellations.” Narratives within constellations reinforce one another; they are made more plausible in the context of the other narratives. It is unlikely that any significant economic event can be causally explained by any single constellation of narratives. Events are rarely so clearly explicable. - -Confluences of narratives, or groups of narratives that do not seem associated with one another but that produce similar effects, can also help drive or explain a significant economic event. - -As Shiller puts it, "new contagious narratives cause economic events, and economic events cause changed narratives." - ---- - -#### Related - -- [[Narratives generate real-world effects]] -- [[Metaphors structure our interpretation of the world]] -- [[¶ Economic narrative]] -- [[§ Writing]] - -#### Reference - -Shiller, Robert J. _Narrative Economics: How Stories Go Viral and Drive Major Economic Events_. Princeton: Princeton University Press, 2019. diff --git "a/topics/communication/\302\266 Pyramid Principle.md" "b/topics/communication/\302\266 Pyramid Principle.md" deleted file mode 100644 index 00ac5d96d..000000000 --- "a/topics/communication/\302\266 Pyramid Principle.md" +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: ¶ pyramid principle -description: Learn how the Pyramid Principle helps organize information by synthesizing data from the bottom up and communicating key insights clearly from the top down for effective persuasive writing. -date: null -redirect: - - /yNlRiA ---- - -The Pyramid Principle, popularized by McKinsey's Barbara Minto, is a framework for organizing information. - -The pyramid has two core principles: - -1. Synthesize from the bottom up -2. Communicate from the top down - -## Synthesize from the bottom up - -The process assumes that you have a clearly defined problem and hypothesis to drive your research. - -You can use the principle to organize your data into three levels: - -1. **Lower level**: Information, analysis, survey results. In other words, raw data. -2. **Middle level**: Synthesized arguments, themes, or takeaways of a group of arguments. In other words, the key insights or patterns that have emerged from the data. Each insight should be composed only of the elements below it in the structure. -3. **Top level**: The main takeaway from the information you have analyzed—the big insight that sums up everything you have learned. - -![20200802 The Pyramid Principle Diagram.png](https://publish-01.obsidian.md/access/5bf4c22f8416d93237aa3630d0fd9c7c/assets/20200802%20The%20Pyramid%20Principle%20Diagram.png) - -At each level, strive to be mutually exclusive and comprehensively exhaustive (MECE). - -## Communicate from the top down - -Present your findings in the reverse order in which you developed them. That is, communicate from the top down. - -Start with the answer. Remember that people can only hold a few "chunks" of data in their head at any one time. - -Consider writing a memo rather than creating a deck, at least to start. [[Writing is thinking]] Begin with the overall recommendation and summarizing the key insights. (This is your topic sentence.) - -Then, address each point in turn. Once again, ensure that your ideas are MECE. Each point would be its own paragraph. - -### Direct storytelling versus indirect storytelling - -Usually, starting with the answer (direct storytelling) is the best approach for persuasive writing. Use this approach when dealing with - -- friendly clients; -- impatient clients; or -- big-picture, strategic discussions in which there is already some acceptance around the proposed recommendations. - -However, if you fear pushback, you may use indirect storytelling to ease the audience into accepting your recommendation. Use the indirect approach for - -- controversial recommendations -- hostile audiences -- analytical organizations and personalities -- audiences who love data but are not impatient - ---- - -#### Citations - -- -- diff --git "a/topics/communication/\342\211\210 Metaphors We Live By.md" "b/topics/communication/\342\211\210 Metaphors We Live By.md" deleted file mode 100644 index 240881004..000000000 --- "a/topics/communication/\342\211\210 Metaphors We Live By.md" +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /sXHRRg ---- - -## Summary - -Metaphors are not simply linguistic flourishes that we use to artfully describe the world around us. They are conceptual tools that help us interpret and even construct the meanings of our experiences and interactions. - -## Notes - -### We use metaphors to interpret and understand the world around us - -- [Metaphors structure our interpretation of the world](https://publish.obsidian.md/mobydiction/notes/Metaphors+structure+our+interpretation+of+the+world) -- [Metaphors create coherent systems of meaning](https://publish.obsidian.md/mobydiction/notes/Metaphors+create+coherent+systems+of+meaning) -- [Structural metaphors reveal and conceal different aspects of experience](https://publish.obsidian.md/mobydiction/notes/Structural+metaphors+reveal+and+conceal+different+aspects+of+experience) -- [Orientational metaphors organize concepts relationally](https://publish.obsidian.md/mobydiction/notes/Orientational+metaphors+organize+concepts+relationally) -- [Metonymy facilitates understanding by focusing our attention](https://publish.obsidian.md/mobydiction/notes/Metonymy+facilitates+understanding+by+focusing+our+attention) -- [Novel metaphors create novel ways of thinking](https://publish.obsidian.md/mobydiction/notes/Novel+metaphors+create+novel+ways+of+thinking) - -### Experientialism suggests we understand the world through our interactions with it - -- [Objectivism assumes a concrete, knowable world](https://publish.obsidian.md/mobydiction/notes/Objectivism+assumes+a+concrete%2C+knowable+world) -- [Subjectivism suggests that our senses and emotions are our best guides to truth](https://publish.obsidian.md/mobydiction/notes/Subjectivism+suggests+that+our+senses+and+emotions+are+our+best+guides+to+truth) -- [Experientialism suggests that truth is relative to our interactions with the world](https://publish.obsidian.md/mobydiction/notes/Experientialism+suggests+that+truth+is+relative+to+our+interactions+with+the+world) - ---- - -#### Reference - -Lakoff, George, and Mark Johnson. _Metaphors We Live By_. University of Chicago Press, 2008. diff --git "a/topics/communication/\342\211\210 Narrative Economics.md" "b/topics/communication/\342\211\210 Narrative Economics.md" deleted file mode 100644 index 0aa5d63c4..000000000 --- "a/topics/communication/\342\211\210 Narrative Economics.md" +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - communication -redirect: - - /vdqnLQ ---- - -## Summary - -Shiller explores how "economic narratives" spread like contagions through societies, ultimately creating real-world economic effects. Economic narratives recur and mutate over time, and spread in unpredictable ways. These can drive individual behaviour around where to invest, how much to save, etc., but also mass-scale attitudes and events, such as the idea that private enterprises provide a model for efficiency and should not be subject to government intervention. - -## Concepts - -- [[¶ Consilience]] -- [[¶ Constellation]] -- [[¶ Flashbulb memory]] - -## Notes - -- [[Economic narratives spread like diseases]] -- [[Narrative helps the human mind make sense of events]] - ---- - -#### Reference - -Shiller, Robert J. _Narrative Economics: How Stories Go Viral and Drive Major Economic Events_. Princeton: Princeton University Press, 2019. diff --git "a/topics/communication/\342\211\210 The Opposable Mind.md" "b/topics/communication/\342\211\210 The Opposable Mind.md" deleted file mode 100644 index 26e473a6f..000000000 --- "a/topics/communication/\342\211\210 The Opposable Mind.md" +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: ≈ the opposable mind -description: Discover how successful leaders use integrative thinking to solve complex problems by combining different perspectives and avoiding simple either/or choices for creative solutions. -date: null -redirect: - - /e20g1A ---- - -## Summary - -The most successful leaders practice what Martin calls "integrative thinking": an approach to problem-solving that embraces complexity and eschews binary, either/or thinking. Integrative thinkers are able to consider the problem holistically, rather than as a series of discrete decisions or trade-offs, and adopt a broad view of salience to develop unique, creative approaches to the problem space. - -## Notes - -### Integrative thinking - -- [[Qualities of an integrative thinker]] -- [[Put mental models in conversation with one another]] -- [Perspective comprises our stance, our tools, and our experience](https://publish.obsidian.md/mobydiction/notes/Perspective+comprises+our+stance%2C+our+tools%2C+and+our+experience) - -### Salience - -- [[Salience is subjective]] -- [Specialization narrows our perspective on what is salient to a problem](https://publish.obsidian.md/mobydiction/notes/Specialization+narrows+our+perspective+on+what+is+salient+to+a+problem) - -#### Citation - -Martin, Roger L. The Opposable Mind: How Successful Leaders Win Through Integrative Thinking. 1st edition. Boston, Mass: Harvard Business Review Press, 2007. diff --git a/topics/data/202210150019-migration-planning.md b/topics/data/202210150019-migration-planning.md deleted file mode 100644 index 701576197..000000000 --- a/topics/data/202210150019-migration-planning.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Migration planning -description: Migration planning is a crucial part of any software development project, especially when migrating to a new database or platform. -date: 2022-10-14 -authors: - - tienan92it -tags: - - migrations -redirect: - - /VpUWag ---- - -https://newsletter.pragmaticengineer.com/p/real-world-engineering-challenges - -tl;dr -**Migration plan** -What do we need to consider? - -- Downtime -- Data consistency -- Rollback plan -- Observable and measurable -- Team awareness - -Common steps - -- Dual reads / writes -> old database is primary -- Backfilling -- Compare and validate the new database -- Dual reads / writes -> new database is primary -- Remove old database diff --git a/topics/data/202211141513-materialized-view-pattern.md b/topics/data/202211141513-materialized-view-pattern.md deleted file mode 100644 index 991433461..000000000 --- a/topics/data/202211141513-materialized-view-pattern.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Materialized view pattern -description: Materialized view is the cache of views. It pre-computes, stores, and optimizes data access when created, and automatically refreshes to ensure real-time data availability. -date: 2022-11-14 -authors: - - haongo138 -tags: - - data - - database - - sql - - data-warehouse - - materialized-view -redirect: - - /xgpj8Q ---- - -**Materialized view pattern** -TL,DR; -Versus "normal" view - -- A normal view provides ease-of-use and flexibility features, but it DOES NOT speed up data access -- A Materialized view is the cache of views. It pre-computes, stores, and optimizes data access when created, and automatically refreshes to ensure real-time data availability - -**Usecases** - -- In data warehouses that have a large number of complex queries on large tables, consume a lot of time & resource, materialized views can eliminate the overhead of expensive joins and aggregations by responding to queries by pre-computed results. -- Especially useful for queries that can be anticipated and repeatedly use the same subquery results. - -**Two main refresh strategies** -1/ Complete refresh - -- Running within one transaction -- At the beginning, the old data of the materialized view is deleted -- Then, the new data is inserted by running the underlying SQL query. -- At the end of the refresh, the transaction is committed, and the new data is visible for all users. - Pros: During this process, users can still use the materialized view and see the old data - Cons: This process can take a long time as the number of rows that the materialized view contains - -2/ Fast refresh (“incremental refresh” would be more appropriate) -In most cases, this method is much faster than a Complete Refresh - -- A fast refresh requires having a materialized view log on each of the source tables that are referenced in the materialized view -- There are several preconditions to enable Fast Refresh, and if only one of them is missing, the Fast Refresh method does not work (can debug by using dbms_mview.explain_mview) - -**References** - -- https://learn.microsoft.com/en-us/azure/architecture/patterns/materialized-view -- https://docs.oracle.com/en/database/oracle/oracle-database/18/dwhsg/refreshing-materialized-views.html#GUID-BB945209-8D69-4FC7-844E-35C9ED7C8A80 - -#data -@brain master diff --git a/topics/data/202301191192-multi-column-index-in-db.md b/topics/data/202301191192-multi-column-index-in-db.md deleted file mode 100644 index 8255faeaf..000000000 --- a/topics/data/202301191192-multi-column-index-in-db.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Multi-column index in DB -description: Multi-column index in DB -date: 2023-01-19 -authors: - - hmhoang13 -tags: - - database - - composite-index - - index - - multi-column-index -redirect: - - /xauIwA ---- - -**Multi-column index in DB** - -Databases support creating composite indexes (index on multiple columns). If you specify the columns in the right order in the index definition, a single composite index can speed up several kinds of queries on the same table. -Suppose that a table has the following specification: - -```sql -CREATE TABLE test ( - id INT NOT NULL, - col1 CHAR(30) NOT NULL, - col2 CHAR(30) NOT NULL, - col3 CHAR(30) NOT NULL, - PRIMARY KEY (id), - INDEX name (col1, col2, col3) -); -``` - -The index name is used for lookups in the following queries: - -```sql -SELECT * FROM tbl_name WHERE col1=val1; -SELECT * FROM tbl_name WHERE col1=val1 AND col2=val2; -SELECT * FROM tbl_name WHERE col1=val1 AND col2=val2 AND col3=val3; -``` - -The index name is not used for lookups in the following queries: - -```sql -SELECT * FROM tbl_name WHERE col2=val2; -SELECT * FROM tbl_name WHERE col2=val2 AND col3=val3; -``` - -**Summary**: If the table has a multiple-column index, any leftmost prefix of the index can be used by the optimizer to look up rows. diff --git a/topics/data/acid-model.md b/topics/data/acid-model.md deleted file mode 100644 index 8536e892f..000000000 --- a/topics/data/acid-model.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: Acid model -description: Learn how the ACID model ensures database transaction reliability with atomicity, consistency, isolation, and durability for accurate, secure, and concurrent data management in ACID-compliant systems. -date: 2022-04-18 -github_id: monotykamary -redirect: - - /HWltPQ ---- - -## What is the ACID model? - -ACID is an acronym that generally describes the necessity for consistency of a transaction in a database. The acronym stands for: - -- **Atomicity**: Each transaction is handled as a unit of work that is either properly carried out or halted. When it is halted, the transaction is reverted to the previous state before the transaction to ensure validity of all data in the database. -- **Consistency**: Although self-explanatory with regard to data, any rules, constraints, cascades, and triggers must have valid and consistent data. -- **Isolation**: Transactions cannot affect or jeopardize the integrity of other transactions by interacting with them while they are still in progress. This means a majority of transactions that are ACID should run concurrently. -- **Durability**: After a commit of a completed transaction, we can be assured that the transaction itself will be persisted in the event of a network partition or a power outage. _This does not assume or take in consideration single-upset events._ - -![](assets/acid-model_acid_acronym_diagram.webp) - -## Why use ACID? - -ACID is particularly important for businesses when there is a high requirement for consistent and durable data. Data consistency or loss of data would eventually translate to loss of revenue, especially if the software in question is critical for daily operations or strategic analysis. Thresholds for requiring an ACID compliant database would be: - -- Requirement in order of transactions and activities -- Low to zero tolerance for incomplete transactions -- Multiple access of processes or users to the database -- Low to zero tolerance for showing stale data to users - -## What is ACID compliance? - -Successfully hitting the requirements for the acronym would essentially give the database ACID compliance. When the database has proof ACID-compliant documents and management, business get to benefit for the insurance of: - -- **Less user disruptions**: When a system or business logic fails to handle a case, database durability should prevent users from noticing big issues. -- **Protected transactions:** ACID compliant transactions regarding fiat or money transfers prevents loss of capital when an operation fails. -- **Consistent accuracy:** ACID compliance ensures a high level of correctness with regard to aggregation of data, such as totals in a bank balance. You can be assured that the total is consistent to the activities that have taken place. -- **Cost savings and reduced risk:** With relevance to finance, ACID transactions prevent double spending or cases of irretrievable capital. -- **Precise timeliness:** Record access of ACID compliant data means that data will always be up-to-date, regardless of what transactions are currently in progress. ACID compliance ensures concurrency control such that, transactions must reach a state that is consistent to the user accessing the data. - -## What databases are ACID compliant? - -Most relational databases are ACID compliant, such as MySQL, PostgreSQL, Oracle, SQLite, and Microsoft SQL Server. NoSQL databases such as Apache's CouchDB, ArangoDB, or IBM's Db2 also implement or follow a close implementation of ACID for compliance. - -#### Reference - -- https://database.guide/what-is-acid-in-databases/ -- https://phoenixnap.com/kb/acid-vs-base -- https://www.indeed.com/career-advice/career-development/acid-database diff --git a/topics/data/apache-hadoop-and-big-data.md b/topics/data/apache-hadoop-and-big-data.md deleted file mode 100644 index 8bbb328d0..000000000 --- a/topics/data/apache-hadoop-and-big-data.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: Apache Hadoop and big data -description: "Learn what Big Data is by exploring the 3 V's—Volume, Velocity, and Variety—and discover how Hadoop provides scalable, cost-effective solutions for storing and processing massive data sets." -date: 2022-06-12 -github_id: dudaka -redirect: - - /yNqyVA ---- - -## What is Big Data? - -### 3 V's: volume, velocity and variety - -#### Volume - -"Big" is a relative term. We put Shaq next to almost everyone, but especially gymnast Simone Biles, and he looks enormous. But if he stands with Yao Ming ... suddenly he doesn't look so big. - -![](assets/apache-hadoop-and-big-data_big-data-illustration.webp) - -The same applies to data. What is considered "Big"? 10GB? 100GB? Or 100TB? There is no straight number that defines big data. There are two reasons why there is no straightforward answer to this question: - -- What is considered as big today, in term of data size/volume, need to be considered as big a year from now. This is a moving target. -- It's all relative, as shown in Shaq's photos. What we consider to be big may not be the case for companies like Google and Facebook. - -Hence, For these two reasons, it's very hard to put a number to define Big Data volume. - -Let's say we are defining big data problems in terms of pure Volume alone. In our opinion, 100GB is not big data since we have a hard disk greater than 100GB. How about 1TB? It is still not because a well-defined traditional database can handle 1TB or even more without any issue. Then, 100TB? Some would claim that 100TB is a big data problem, and others might disagree. It is relative! So, 1000TB? Now, this is on a scale of Petabytes, and it is definitely Big Data. - -#### Velocity - -We have to understand that the volume of data is not the only factor in classifying our data be big data or not. Let's say we work at a start-up, and we recently launched a very successful email service where users can log in to send and receive emails. Our email service is so good and even better than Gmail. In three months, we have 100 thousand active users signed up to use our service. - -Hypothetically, we are currently using a traditional database to store messages and their attachments, and also our current size of the database is 1TB. So, do we have a big data problem? The straightforward answer is NO because 1TB is not that big to classify as a big data problem. Another question: in this growth rate, will we have a big data problem in the near future? To answer this, we need to consider three factors. - -The first one is volume. In 3 months, our start-up has 100 thousand active users, and our volume is 1TB. If we have a positive growth at the same rate, we will have 400 thousand active users at the end of the year, and our volume will be 4TB. What if we doubled or tripled our user base every three months, so the bottom line is that we should not just look at the volume when we think of Big Data. We should look at the rate at which our data grows. In other words, we should watch the velocity or speed of our data growth. - -Velocity is the next important factor to consider; it tells us how fast our data is growing. If your data volume stays at 1TB for a year, all we need is a good database. If your growth rate is 1TB/week, then you have to think about a scalable Big Data solution. Most of the time, Volume and Velocity are all you need to decide whether you have a Big Data problem or not. - -#### Variety - -This is the next factor we need to consider; it adds one more dimension. Our data and traditional database are highly structured, which are rows and columns. Back to our hypothetical start-up email service, it receives data in various formats: texts for the actual messages, images, and videos as attachments. When we have data coming to our system in different forms and have to process or analyze the data in different formats, traditional database systems are sure to fit. When combined with high volume and velocity, you for sure have a big data problem. - -Therefore, whenever we are asked whether it is a big data problem or not, please take the 3 V's: Volume, Velocity, and Variety into consideration. This happens to Big Data consultants all the time; they will be called in by clients about data storage that has performance issues and hope that a Big Data solution like Hadoop is going to solve their problem. Most of the time, their answer will fail in the volume and velocity tests, the volume will be in the higher gigabytes or lower gigabytes, and their growth rate has been relatively low for the past six months and in the foreseeable future. Hence, the volume does not qualify as big data, and their data growth rate will be very low. It fails the velocity test as well. What the client needs is to optimize the existing process and not a sophisticated Big Data solution. - -### Usecases - -When we say Big Data, we are potentially talking about hundreds to thousands of Terabytes. Let's consider the following domains' use cases: - -#### Science - -Large Hadron Collider at SUN produces about 1TB of data every second, mostly sensor data from their equipment. Their volume is so big, that they don't even retain or store all the data they produce. - -NASA gathers about 1.73 GB of data every hour about geolocation data from satellites, etc. - -#### Goverment - -NSA (National Security Agency) is known for its controversial data collection programs. An NSA data center in Utah can house 1 Yottabyte (1 trillion terabytes) of data in terms of Volume. - -In March 2012, Obama's administration announced about 200 million dollars in Big Data initiatives. We can understand the significance behind Big Data and its analysis even though we cannot technically classify the next one under a government. And Obama's 2nd term election campaign used Big Data analytics which gave them a Competitive Edge to actually win the election. - -#### Private - -With the advent of social media like Facebook, Twitter, LinkedIn, etc., there is no scarcity of data: eBay is known to have a 40 Petabyte cluster and Facebook a 30 Petabyte cluster. These numbers are old now since the stats are a little old big data. - -Data is not only produced and analyzed in social media companies but also retail space. It is most common in several major retail websites to capture click-stream data. For example, you shop at amazon.com. Amazon is not only capturing data when you click checkout but also every click on their website, which is tracked to bring a personalized shopping experience. When Amazon shows you recommendations, Big Data analytics is at work behind the scenes. - -### Big data challenges - -Big Data comes with big problems: - -- Since data sets are huge, we need to find a way to store them as efficiently as possible. It is not just about efficiency in terms of \***\*storage\*\*** space but also efficiency in storing the data set that is suitable for \***\*computation\*\***. The main purpose of storing data is to analyze them, right? How much time does it take to analyze and provide a solution to a problem using our big data? What's good in storing the data when you cannot analyze or process the data in a reasonable time? With big data set, computation with reasonable execution times is a challenge - -- Another problem when we deal with big data set is \***\*data loss\*\*** due to corruption and data or due to hardware failure. You need to have a proper recovery strategy in place. - -- Finally, the \***\*cost\*\*** and the most important challenge you're going to need a lot of storage space and a lot of computational power. Therefore, the solution that you plan to use should be cost-effective. - -### Traditional solutions - -#### RDBMS - -Traditional RDBMS will have scalability issues when moving up in data volume in terms of Terabytes. We will be forced to demoralize and pre-aggregate the data for faster query execution time. As the data get bigger, we will be forced to make changes to the process in terms of changing the indexes, optimizing the queries, etc. Assuming that your database is running with enough hardware resources, when you see a performance issue, you still have to make changes to the query itself or the way in which your data is accessed. There is no working around it. You cannot add more hardware resources or more computer nodes and distribute the problem to bring the computation time down. In other words, the database is not horizontally scalable, i.e., you cannot add more resources or more computation nodes and hope the execution time or the performance will improve. - -Databases are designed to process structured data. When our data does not have a proper structure, the database will struggle. Furthermore, a database is not a good choice when you have a variety of data which is data in several formats like texts, images, videos, etc. - -A good enterprise-grade database solution can be quite expensive for a relatively low volume of data when you add hardware costs and platinum-grade storage costs. It's going to be quite expensive. - -#### Grid computing - a distributed computation solution - -Grid computing is essentially many nodes operating on data parallelly and then doing faster computation. However, there are two challenges: - -- Grid or high-performance computing is good for computing-intensive tasks with a relatively low volume of data but does not perform well when the data volume is huge. -- Grid computing requires a good experience with lower-level programming to implement and then it is not suitable for the mainstream. - -### Hadoop - a good solution - -A good solution should, of course, handle a huge volume of data. It should provide efficient storage, which is the ability to store data efficiently. Data loss is unavoidable, so the proposed solution should implement a good recovery strategy. And the solution should be horizontally scalable as your data grows. Most importantly, it should be cost-effective. Finally, to minimize the learning curve, it should be easy for programmers, data analysts, and non-programmers to work with the framework or the system. This is exactly what Hadoop offers. - -#### Is Hadoop a replacement for RDBMS? - -NO! - -| Hadoop | RDBMS | -| --------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Hadoop has the volume in terms of petabytes | RDBMS works exceptionally well with volume in low terabytes | -| Hadoop can work with Dynamic Schema and supports files in many different formats | The schema is very strict and not so flexible and cannot handle multiple formats | -| Hadoop solution can scale horizontally | RDBMS's solution can scale vertically, meaning we can add more resources to the existing solution and to make any improvements to the process itself like tuning the queries and adding more indexes, etc. | -| Hadoop offers a cost-effective solution | It gets expensive very quickly when we increase the volume of data | -| Hadoop is a batch processing system, so we cannot expect a millisecond response time like an interactive system | RDBMS is an interactive and batch system | -| We can write the file or data once and then operate or analyze data multiple times | we can read and write multiple times | - -The gaps between Hadoop and RDBMS are closing in. Hadoop offers a cost-effective solution to big data problems, but Hadoop is not the only solution that is available in the market now. NoSQL databases like HBase and Cassandra bring a great deal of value in analyzing a huge volume of data, and it is a great alternative for RDBMS. Now, when we mention a huge volume of data, we are talking about millions of columns and billions of rows. - -## Understanding big data problem - -### Sample big data problem - -Imagine you work at one of the major exchanges like the New York Stock Exchange or NASDAQ. One morning someone from your Risk Department stops by your desk and asks you to calculate the maximum closing price of every stock symbol that is ever traded in the exchange since inception. Also, assume the size of the data set you are given is 1 TB, so your data set would look like the below image: - -![](assets/apache-hadoop-and-big-data_sample-big-data-problem.webp) - -Each line in this data set is information about a stock for a given date. Immediately the business user who gave this problem asks you for an ETA on when he can expect the results. There is a lot to think about here, so you ask him to give you some time, and you start to work. What would be your next steps? You have two things to figure out: storage and computation. - -Let's consider the storage first. Your workstation has only 20 GB of free space, but the size of the data set is 1 TB. Thus, you go to your storage team and ask them to copy the data set to a NAS (Network Attached Storage) server or even a SAN (Storage Area Network) server. Once the data set is copied, you ask them to give you the location of the data set. Because a NASA or san is connected to your network, any computer with access to the network can access the data provided if their permission to see the data. Thus, the data is stored, and you have access to the data. - -Now, the next problem is computation. You're a Java programmer, so you wrote an optimized Java program to parse the data set and perform the computation. And you're now ready to execute the program against the data set. Unfortunately, you realize it's already noon, the business user who gave you this request stopped by for an ETA. Then, you start to think what is the ETA for this whole operation to complete, and you come up with the result set. - -### Execution time - -For the program to work on the data set, first, the data set needs to be copied from the storage to the working memory or Ram. How long does it take to copy a one-terabyte data set from storage? Let's take our traditional hard disk drive, which is the one that is connected to a laptop or workstation, etc. HDDs (Hard Disk Drive) have magnetic platters in which the data is stored. When you request to read data, the head in the hard disk first position itself on the platter and starts transferring the data from the platter to the head. The speed at which the data is transferred from the platter to the head is called the data access rate. Average data access rates in HDDs are usually about 122 MBs. So, to read 1 TB from an HDD, you need 2 hours and 22 minutes. That is for an HDD that is connected to your workstation. - -When you transfer a file from a NAS server or from your SAN server, you should know the transfer rate of the hard disk drives in the NAS servers. For now, we will assume it is the same as the regular HDD, which is 122 MBs, and hence it would take 2 hours and 22 minutes. Next, what about the computation time? Since you have not executed the program yet at least once, you cannot say for sure. Additionally, your data comes from a storage server that is attached to the network, and you have to consider the network bandwidth also. With all that in mind, you give him an ETA of about three hours, but it could be easily over three hours since you're not sure about the computation time. - -```sh -Data Access Rate + Program Computation Time (~60 min) + Network Bandwidth, etc. = (>3 hours) -``` - -Unsurprisingly, your business user is so shocked to hear three hours for an ETA he has the next question: can we get it sooner than three hours? Maybe in 30 minutes. You know there is no way you can execute the results in 30 minutes. Of course, the business cannot wait for three hours, especially in finance, for time is money. - -How can we calculate the result in less than 30 minutes? The majority of the time you spend calculating the result set will be attributed to two tasks: - -- Transferring the data from storage or hard disk drive is about two and a half hours and -- The computation time is the time to perform the actual calculation by your program (~ 60 minutes), it could be more, or it could be less. - -What if we replace HDDs with SSDs (Solid State Drives)? SSDs are a very powerful alternative for HDD. SSD does not have magnetic platters, heads, or any moving components. And, it's based on flash memory, so it's extremely fast. By doing that, we can significantly reduce the time it would take to read the data from the storage. But here's the problem SSD comes with a price usually five to six times the price of your regular HDD. Although the price continues to go down, given the data volume that we are talking about with respect to Big Data, it is not a viable option. Therefore, for now, we are stuck with HDDs. - -Let's consider how we can reduce the computation time. Hypothetically, the program will take 60 minutes to complete and is also already optimized for execution. We can divide the 1TB data set into 100 equal size chunks/blocks and have 100 computers/nodes to the computation parallelly. Theoretically, we cut the data access by the factor of 100 as well as the computation time. Hence, the data access time is reduced to less than 2 minutes (= 142 mins / 100), and the computation time is in less than 1 minute (= 60 mins / 100). - -![](assets/apache-hadoop-and-big-data_1tb-of-big-data.webp) - -Furthermore, if you have more than one chunk of your data set stored in the same hard drive, you cannot get a true parallel read because there is only one head in your hard disk which does the actual read. For the sake of argument, assuming that we get a true parallel read, which means we have 100 nodes trying to read data at the same time. Assuming the data can be read parallelly, we will now have (100 x 122) MBs of data flowing through the network. Imagine what would happen when each one of your family members at home starts to stream their favorite TV show or movie at the same time using a single internet connection at your home? It would result in a very poor streaming experience with a lot of buffering such that no one in the family can enjoy their show. What we have essentially done is choked up your network; the download speed is requested by each one of the devices combinedly exceeded the download speed offered by the internet connection resulting in poor service. This is what will happen when 100 nodes try to transfer the data over the network at the same time. - -![](assets/apache-hadoop-and-big-data_100-nodes-of-big-data.webp) - -To solve this, we can bring the data closer to the computation, i.e., store the data locally on each node's hard disk. Thus, you would store Block 1 of data in Node 1, block 2 of data in node 2, etc., as shown in the above image. Now we can achieve a true parallel read on all 100 nodes, and also, we have eliminated the network bandwidth issue. That's a significant improvement or design. - -How can we protect our data from hard disk failure or data loss, data corruption, etc. ? For example, you have a photo of your loved ones, and you treasure that picture. In your mind, there is no way you can lose the picture; how would you protect it? You would keep copies of your picture in different places, maybe one on your personal laptop, one copy in Picasa, one copy on your external hard drive, etc. If your laptop crashes, you can still get that picture from Picasa or your external hard drive. From the idea, we copy each block of data to two more nodes. In other words, we can replicate the block in two more nodes. In total, we have three copies of each block. - -![](assets/apache-hadoop-and-big-data_block-of-data-notes.webp) - -As shown in the above image, Node 1 has Block 1, 7, and 10. Node 2 has Blocked 7, 11, and 42. Node 3 has blocks 1, 7, and 10. If block one is unavailable in Node 2 due to a hard disk failure or corruption in the block, it can be easily fetched From node 3. This means that Node 1, 2, and 3 must have access to one another, and they should be connected to a network. But there are some challenges in implementing it; how does Node 1 know that Node 3 has Block 1? And who decides Block 7, for instance, should be stored in Node 1, 2, and 3. First of all, who will break the 1TB into 100 blocks? - -That's just the storage part; computation brings other challenges. Node 1 can only compute the maximum close price from just Block 1. Similarly, Node 2 can only compute the maximum close price from Block 2. This brings up a problem because, for example, data for stock GE (a stock symbol) can be in Block 1 and can also be in Block 2 and could also be on block 82, for instance, right. Then, you have to consolidate the result from all the nodes together to compute the final result; who is going to coordinate all that? The solution we are proposing is distributed computing, and as we are seeing, there are several complexities involved in implementing the solution both at the storage layer and also at the computation layer. - -![](assets/apache-hadoop-and-big-data_hadoop-infrastructure.webp) - -The answer to all these open questions and complexities is Hadoop. Hadoop offers a framework for distributed computing. Hadoop has two core components, HDFS and MapReduce. HDFS stands for Hadoop Distributed File System, and it takes care of all your storage-related complexities like splitting your data set into blocks, replicating each block to more than one node, and also keep track of which block is stored on which node, etc. MapReduce is a programming model, and Hadoop implements MapReduce, and it takes care of all the computational complexities. Therefore, Hadoop framework takes care of bringing all the intermediate results from every single node to offer a consolidated output. - -What is Hadoop? Hadoop is a framework for distributed processing of large data sets across clusters of commodity computers The last two words in the definition are what makes Hadoop even more special, commodity computers, which means all the hundred nodes that we have in the cluster do not have to have any specialized hardware, i.e., their enterprise-grade server nodes with the processor, hard disk and RAM in each of them. There's nothing more special about that. But don't confuse commodity computers with cheap hardware. Commodity computers mean inexpensive hardware and not cheap hardware. - -Now, you know what Hadoop is and how it can offer an efficient solution to your maximum close price problem against the 1TB data set. So, you can go back to the business and propose Hadoop to solve the problem and to achieve the execution time that your users are expecting. But if you propose a 100 node cluster to your business, expect to get some crazy looks; that's the beauty :)). - -You don't need to have a 100 node cluster; we have seen successful Hadoop production environments from small 10 node cluster all the way to 100 to 1000 node cluster. You can simply even start with a 10 node cluster, and if you want to reduce the execution time even further, all you have to do is add more nodes to your cluster. That's simple. In other words, Hadoop will scale horizontally. - -## History of Hadoop - -![](assets/apache-hadoop-and-big-data_history-of-hadoop.webp) - -In 2002, an excellent and smart programmer named Doug Cutting was working on an open-source project named Nutch. The purpose of the nudge is to crawl the internet, collect web pages then rank and index them so that we can run searches against the indexed web pages. That is exactly what Google's search engine does. Google's proprietary crawlers and algorithms crawl through the internet, collect web page ranks and index them and run searches against them. Nutch is a very similar project to that idea, but it's open-source. - -Soon, Doug Cutting realized that he was hitting scalability issues both in terms of storage and in terms of computation because he was collecting and trying to analyze a massive amount of data. While he was trying to solve the scalability issues, in 2003, Google published a paper about their proprietary homegrown file system named Google File System (GFS) and how it can be used to store massive amounts of data and offer redundancy and scalability at the same time. And in 2004, Doug created an open-source version of GFS and called it Nutch Distributed File System (NDFS). In 2004, Google published a paper on a programming model named MapReduce that addressed how Google achieved computational efficiency with Big Data. That is what Doug was looking for in terms of computation. And in 2005, Doug managed to run Nutch on top of NDFS and open-source implementation of MapReduce. - -In Feb 2006, a sub-project named Hadoop was founded. Hadoop is the name of Doug's kids - A yellow stuffed elephant doll. He chose to use the name because it was easy to spell and meaningless. Around 2006, Yahoo started funding the efforts by Doug Cutting and building Hadoop and hired Doug. On Feb 19, 2008, Yahoo announced that they have the world's largest Hadoop production cluster. And in Jan 2008, Hadoop was made a top-level Apache project. In April 2008, Hadoop broke the world record and soldered a terabyte of data in 209 seconds. Later that year, in November, Google broke that same record and sorted a terabyte of data in just 68 seconds. Since its inception to date, Hadoop and its community have grown leaps and bounds and were soon embraced by several companies as a solution to analyzing Big Data. - -Hadoop was truly given the ability to analyze volumes and volumes of data and helped unlock key valuable insights that would go unfound if it wasn't for this awesome technology. More importantly, Hadoop brought massive parallel distributed computing to the mainstream since it is open-source and can be implemented using commodity computers. And since it's open-source and can be implemented using commodity computers, Hadoop sets the bar very low in terms of cost of entry which enables even smaller companies to work with and analyze big data. - -## References - -- https://hadoop.apache.org/ -- https://en.wikipedia.org/wiki/Apache_Hadoop -- [Hadoop: the definitive guide: storage and analysis at internet scale](https://www.amazon.com/Hadoop-Definitive-Storage-Analysis-Internet/dp/1491901632/ref=sr_1_2?crid=2LTQHKE9WNBNC&keywords=Hadoop&qid=1657604708&sprefix=hadoop%2Caps%2C127&sr=8-2) diff --git a/topics/data/assets/2023-november-forward-engineering_forward-engineering-november-2023_november-forward-engineering-2023-20231130165019170.webp b/topics/data/assets/2023-november-forward-engineering_forward-engineering-november-2023_november-forward-engineering-2023-20231130165019170.webp deleted file mode 100644 index d18e1d58f..000000000 Binary files a/topics/data/assets/2023-november-forward-engineering_forward-engineering-november-2023_november-forward-engineering-2023-20231130165019170.webp and /dev/null differ diff --git a/topics/data/assets/a-lens-to-modern-data-engineering-orchestra.png b/topics/data/assets/a-lens-to-modern-data-engineering-orchestra.png deleted file mode 100644 index 5da39ac45..000000000 Binary files a/topics/data/assets/a-lens-to-modern-data-engineering-orchestra.png and /dev/null differ diff --git a/topics/data/assets/a-lens-to-modern-data-engineering.png b/topics/data/assets/a-lens-to-modern-data-engineering.png deleted file mode 100644 index 71cf771ce..000000000 Binary files a/topics/data/assets/a-lens-to-modern-data-engineering.png and /dev/null differ diff --git a/topics/data/assets/acid-model_acid_acronym_diagram.webp b/topics/data/assets/acid-model_acid_acronym_diagram.webp deleted file mode 100644 index 8dddcf860..000000000 Binary files a/topics/data/assets/acid-model_acid_acronym_diagram.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_100-nodes-of-big-data.webp b/topics/data/assets/apache-hadoop-and-big-data_100-nodes-of-big-data.webp deleted file mode 100644 index 264690ccb..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_100-nodes-of-big-data.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_1tb-of-big-data.webp b/topics/data/assets/apache-hadoop-and-big-data_1tb-of-big-data.webp deleted file mode 100644 index 4db02c63a..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_1tb-of-big-data.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_big-data-illustration.webp b/topics/data/assets/apache-hadoop-and-big-data_big-data-illustration.webp deleted file mode 100644 index cbd8ded01..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_big-data-illustration.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_block-of-data-notes.webp b/topics/data/assets/apache-hadoop-and-big-data_block-of-data-notes.webp deleted file mode 100644 index cd1c8852e..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_block-of-data-notes.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_hadoop-infrastructure.webp b/topics/data/assets/apache-hadoop-and-big-data_hadoop-infrastructure.webp deleted file mode 100644 index d03211a6b..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_hadoop-infrastructure.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_history-of-hadoop.webp b/topics/data/assets/apache-hadoop-and-big-data_history-of-hadoop.webp deleted file mode 100644 index f47199da7..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_history-of-hadoop.webp and /dev/null differ diff --git a/topics/data/assets/apache-hadoop-and-big-data_sample-big-data-problem.webp b/topics/data/assets/apache-hadoop-and-big-data_sample-big-data-problem.webp deleted file mode 100644 index 5c287c052..000000000 Binary files a/topics/data/assets/apache-hadoop-and-big-data_sample-big-data-problem.webp and /dev/null differ diff --git a/topics/data/assets/base-model_base_model_diagram.webp b/topics/data/assets/base-model_base_model_diagram.webp deleted file mode 100644 index d8ec59ef1..000000000 Binary files a/topics/data/assets/base-model_base_model_diagram.webp and /dev/null differ diff --git a/topics/data/assets/behind-a-hive-table_describe-formtted-stocks.webp b/topics/data/assets/behind-a-hive-table_describe-formtted-stocks.webp deleted file mode 100644 index a7708a6bb..000000000 Binary files a/topics/data/assets/behind-a-hive-table_describe-formtted-stocks.webp and /dev/null differ diff --git a/topics/data/assets/behind-a-hive-table_hive-site.xml.webp b/topics/data/assets/behind-a-hive-table_hive-site.xml.webp deleted file mode 100644 index 5f5522486..000000000 Binary files a/topics/data/assets/behind-a-hive-table_hive-site.xml.webp and /dev/null differ diff --git a/topics/data/assets/behind-a-hive-table_serde-reading-data.webp b/topics/data/assets/behind-a-hive-table_serde-reading-data.webp deleted file mode 100644 index c4807eb49..000000000 Binary files a/topics/data/assets/behind-a-hive-table_serde-reading-data.webp and /dev/null differ diff --git a/topics/data/assets/behind-a-hive-table_serde-writing-data.webp b/topics/data/assets/behind-a-hive-table_serde-writing-data.webp deleted file mode 100644 index e110a29e1..000000000 Binary files a/topics/data/assets/behind-a-hive-table_serde-writing-data.webp and /dev/null differ diff --git a/topics/data/assets/bloom-filter_bloom_filter.webp b/topics/data/assets/bloom-filter_bloom_filter.webp deleted file mode 100644 index 8c210a5c1..000000000 Binary files a/topics/data/assets/bloom-filter_bloom_filter.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_bucketingdemo.webp b/topics/data/assets/buckets-on-apache-hive_bucketingdemo.webp deleted file mode 100644 index 5f61db719..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_bucketingdemo.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_insertdata.webp b/topics/data/assets/buckets-on-apache-hive_insertdata.webp deleted file mode 100644 index 6f8e778fc..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_insertdata.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_insertdone.webp b/topics/data/assets/buckets-on-apache-hive_insertdone.webp deleted file mode 100644 index 95ffaa44a..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_insertdone.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_listofpartitions.webp b/topics/data/assets/buckets-on-apache-hive_listofpartitions.webp deleted file mode 100644 index 4ce362166..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_listofpartitions.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_lsapartition.webp b/topics/data/assets/buckets-on-apache-hive_lsapartition.webp deleted file mode 100644 index 8f7c39611..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_lsapartition.webp and /dev/null differ diff --git a/topics/data/assets/buckets-on-apache-hive_lsbucketing.webp b/topics/data/assets/buckets-on-apache-hive_lsbucketing.webp deleted file mode 100644 index bdece0a12..000000000 Binary files a/topics/data/assets/buckets-on-apache-hive_lsbucketing.webp and /dev/null differ diff --git a/topics/data/assets/cap-theorem_cap_theorem_diagram.webp b/topics/data/assets/cap-theorem_cap_theorem_diagram.webp deleted file mode 100644 index bb1609b60..000000000 Binary files a/topics/data/assets/cap-theorem_cap_theorem_diagram.webp and /dev/null differ diff --git a/topics/data/assets/circular_project.png b/topics/data/assets/circular_project.png deleted file mode 100644 index f70556a33..000000000 Binary files a/topics/data/assets/circular_project.png and /dev/null differ diff --git a/topics/data/assets/circular_purchasing.png b/topics/data/assets/circular_purchasing.png deleted file mode 100644 index a06eebc37..000000000 Binary files a/topics/data/assets/circular_purchasing.png and /dev/null differ diff --git a/topics/data/assets/circular_student.png b/topics/data/assets/circular_student.png deleted file mode 100644 index 2214b56bb..000000000 Binary files a/topics/data/assets/circular_student.png and /dev/null differ diff --git a/topics/data/assets/creating-a-fully-local-search-engine-on-memo-search.webp b/topics/data/assets/creating-a-fully-local-search-engine-on-memo-search.webp deleted file mode 100644 index 429bd69b9..000000000 Binary files a/topics/data/assets/creating-a-fully-local-search-engine-on-memo-search.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_14e655f71283c0bb4e3f9b48de8f69a0_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_14e655f71283c0bb4e3f9b48de8f69a0_md5.webp deleted file mode 100644 index 948a2c88c..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_14e655f71283c0bb4e3f9b48de8f69a0_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_18cef87bd2e06e47fc4c26388b3b6897_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_18cef87bd2e06e47fc4c26388b3b6897_md5.webp deleted file mode 100644 index 0c7707e4d..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_18cef87bd2e06e47fc4c26388b3b6897_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_3aca3432d1627444ce28b323e9773b65_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_3aca3432d1627444ce28b323e9773b65_md5.webp deleted file mode 100644 index 869321ab5..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_3aca3432d1627444ce28b323e9773b65_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_3d7425b1a0728a1b0bb3464749880f6a_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_3d7425b1a0728a1b0bb3464749880f6a_md5.webp deleted file mode 100644 index bca344bfa..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_3d7425b1a0728a1b0bb3464749880f6a_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_58bf592103e00d6b6caa5eead9bd4310_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_58bf592103e00d6b6caa5eead9bd4310_md5.webp deleted file mode 100644 index a170f1a40..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_58bf592103e00d6b6caa5eead9bd4310_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_7d514b708e3dd2bff8747f1a3adb3001_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_7d514b708e3dd2bff8747f1a3adb3001_md5.webp deleted file mode 100644 index 05b8c5ee2..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_7d514b708e3dd2bff8747f1a3adb3001_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_8e94c2b1eae7bec4aa0736af3576cca7_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_8e94c2b1eae7bec4aa0736af3576cca7_md5.webp deleted file mode 100644 index 7ae3cd875..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_8e94c2b1eae7bec4aa0736af3576cca7_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_b51792aac952213b495c5a808756aaea_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_b51792aac952213b495c5a808756aaea_md5.webp deleted file mode 100644 index 3b544d964..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_b51792aac952213b495c5a808756aaea_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-analyst-in-retail-trading_dfe1dcf3b249f0a1569997f9ba5a8f66_md5.webp b/topics/data/assets/data-analyst-in-retail-trading_dfe1dcf3b249f0a1569997f9ba5a8f66_md5.webp deleted file mode 100644 index 666642728..000000000 Binary files a/topics/data/assets/data-analyst-in-retail-trading_dfe1dcf3b249f0a1569997f9ba5a8f66_md5.webp and /dev/null differ diff --git a/topics/data/assets/data-engineering-moc.webp b/topics/data/assets/data-engineering-moc.webp deleted file mode 100644 index 9dc4f9d57..000000000 Binary files a/topics/data/assets/data-engineering-moc.webp and /dev/null differ diff --git a/topics/data/assets/design-file-sharing-system_3.webp b/topics/data/assets/design-file-sharing-system_3.webp deleted file mode 100644 index 199c623af..000000000 Binary files a/topics/data/assets/design-file-sharing-system_3.webp and /dev/null differ diff --git a/topics/data/assets/design-file-sharing-system_4.webp b/topics/data/assets/design-file-sharing-system_4.webp deleted file mode 100644 index b01273d5b..000000000 Binary files a/topics/data/assets/design-file-sharing-system_4.webp and /dev/null differ diff --git a/topics/data/assets/design-file-sharing-system_5.webp b/topics/data/assets/design-file-sharing-system_5.webp deleted file mode 100644 index 9034208ce..000000000 Binary files a/topics/data/assets/design-file-sharing-system_5.webp and /dev/null differ diff --git a/topics/data/assets/design-file-sharing-system_6.webp b/topics/data/assets/design-file-sharing-system_6.webp deleted file mode 100644 index a36c516f4..000000000 Binary files a/topics/data/assets/design-file-sharing-system_6.webp and /dev/null differ diff --git a/topics/data/assets/designing-a-model-with-dynamic-properties-20240820225604474.webp b/topics/data/assets/designing-a-model-with-dynamic-properties-20240820225604474.webp deleted file mode 100644 index 6faaf7792..000000000 Binary files a/topics/data/assets/designing-a-model-with-dynamic-properties-20240820225604474.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120201050.webp b/topics/data/assets/google-data-fusion-20240122120201050.webp deleted file mode 100644 index bc7dc0113..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120201050.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120216075.webp b/topics/data/assets/google-data-fusion-20240122120216075.webp deleted file mode 100644 index 9a28b880e..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120216075.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120312319.webp b/topics/data/assets/google-data-fusion-20240122120312319.webp deleted file mode 100644 index e4ad51d3e..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120312319.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120335588.webp b/topics/data/assets/google-data-fusion-20240122120335588.webp deleted file mode 100644 index 1e1ef5078..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120335588.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120406512.webp b/topics/data/assets/google-data-fusion-20240122120406512.webp deleted file mode 100644 index 0a846adff..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120406512.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120426619.webp b/topics/data/assets/google-data-fusion-20240122120426619.webp deleted file mode 100644 index 136348b95..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120426619.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120526652.webp b/topics/data/assets/google-data-fusion-20240122120526652.webp deleted file mode 100644 index aa596f008..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120526652.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120552908.webp b/topics/data/assets/google-data-fusion-20240122120552908.webp deleted file mode 100644 index 8db5089cb..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120552908.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120615991.webp b/topics/data/assets/google-data-fusion-20240122120615991.webp deleted file mode 100644 index f8002657f..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120615991.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120647202.webp b/topics/data/assets/google-data-fusion-20240122120647202.webp deleted file mode 100644 index 7751e0c78..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120647202.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120708508.webp b/topics/data/assets/google-data-fusion-20240122120708508.webp deleted file mode 100644 index 521433715..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120708508.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120734801.webp b/topics/data/assets/google-data-fusion-20240122120734801.webp deleted file mode 100644 index 9d7232f6c..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120734801.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120800866.webp b/topics/data/assets/google-data-fusion-20240122120800866.webp deleted file mode 100644 index 90bba36f4..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120800866.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120820157.webp b/topics/data/assets/google-data-fusion-20240122120820157.webp deleted file mode 100644 index 7988d5b29..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120820157.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120841121.webp b/topics/data/assets/google-data-fusion-20240122120841121.webp deleted file mode 100644 index 49c5e17da..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120841121.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120859634.webp b/topics/data/assets/google-data-fusion-20240122120859634.webp deleted file mode 100644 index 079a92c25..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120859634.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120943004.webp b/topics/data/assets/google-data-fusion-20240122120943004.webp deleted file mode 100644 index 5b334867d..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120943004.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122120958563.webp b/topics/data/assets/google-data-fusion-20240122120958563.webp deleted file mode 100644 index 67660c89c..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122120958563.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122121014344.webp b/topics/data/assets/google-data-fusion-20240122121014344.webp deleted file mode 100644 index b13b92486..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122121014344.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122121132342.webp b/topics/data/assets/google-data-fusion-20240122121132342.webp deleted file mode 100644 index fe368435f..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122121132342.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122121149587.webp b/topics/data/assets/google-data-fusion-20240122121149587.webp deleted file mode 100644 index 3e01ed734..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122121149587.webp and /dev/null differ diff --git a/topics/data/assets/google-data-fusion-20240122121206926.webp b/topics/data/assets/google-data-fusion-20240122121206926.webp deleted file mode 100644 index 9130886c5..000000000 Binary files a/topics/data/assets/google-data-fusion-20240122121206926.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130334062.webp b/topics/data/assets/google-dataproc-20240122130334062.webp deleted file mode 100644 index d3a4336fb..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130334062.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130408379.webp b/topics/data/assets/google-dataproc-20240122130408379.webp deleted file mode 100644 index c4b070f6a..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130408379.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130427570.webp b/topics/data/assets/google-dataproc-20240122130427570.webp deleted file mode 100644 index 40168109a..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130427570.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130457028.webp b/topics/data/assets/google-dataproc-20240122130457028.webp deleted file mode 100644 index c3392757f..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130457028.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130525347.webp b/topics/data/assets/google-dataproc-20240122130525347.webp deleted file mode 100644 index cc630d43b..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130525347.webp and /dev/null differ diff --git a/topics/data/assets/google-dataproc-20240122130542001.webp b/topics/data/assets/google-dataproc-20240122130542001.webp deleted file mode 100644 index d850bfb86..000000000 Binary files a/topics/data/assets/google-dataproc-20240122130542001.webp and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_15-crashes.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_15-crashes.png deleted file mode 100644 index 8e2bf1be3..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_15-crashes.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_3-crashes.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_3-crashes.png deleted file mode 100644 index 2689d156d..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_3-crashes.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_copy-operation.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_copy-operation.png deleted file mode 100644 index bfcd40f39..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_copy-operation.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_file-size-vs-block-size.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_file-size-vs-block-size.png deleted file mode 100644 index c851166e5..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_file-size-vs-block-size.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_file-system-hdfs.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_file-system-hdfs.png deleted file mode 100644 index 3ae69e7f9..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_file-system-hdfs.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-distributed-file-system.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-distributed-file-system.png deleted file mode 100644 index 3e18a86b9..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-distributed-file-system.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-nodes.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-nodes.png deleted file mode 100644 index 779228d3f..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_hadoop-nodes.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_node-proximity.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_node-proximity.png deleted file mode 100644 index de9a4ac37..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_node-proximity.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_read-operation.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_read-operation.png deleted file mode 100644 index da2c415fc..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_read-operation.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_replica-placement.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_replica-placement.png deleted file mode 100644 index d3196d389..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_replica-placement.png and /dev/null differ diff --git a/topics/data/assets/hadoop-distributed-file-system-hdfs_write-operation.png b/topics/data/assets/hadoop-distributed-file-system-hdfs_write-operation.png deleted file mode 100644 index d466c842c..000000000 Binary files a/topics/data/assets/hadoop-distributed-file-system-hdfs_write-operation.png and /dev/null differ diff --git a/topics/data/assets/hive-window-and-analytic-functions_apple-ma-10.webp b/topics/data/assets/hive-window-and-analytic-functions_apple-ma-10.webp deleted file mode 100644 index 79f92b781..000000000 Binary files a/topics/data/assets/hive-window-and-analytic-functions_apple-ma-10.webp and /dev/null differ diff --git a/topics/data/assets/hive-window-and-analytic-functions_frames.webp b/topics/data/assets/hive-window-and-analytic-functions_frames.webp deleted file mode 100644 index 2c4d77a5c..000000000 Binary files a/topics/data/assets/hive-window-and-analytic-functions_frames.webp and /dev/null differ diff --git a/topics/data/assets/hive-window-and-analytic-functions_select-result-analytic-function.webp b/topics/data/assets/hive-window-and-analytic-functions_select-result-analytic-function.webp deleted file mode 100644 index d99e42ad2..000000000 Binary files a/topics/data/assets/hive-window-and-analytic-functions_select-result-analytic-function.webp and /dev/null differ diff --git a/topics/data/assets/hive-window-and-analytic-functions_select-result.webp b/topics/data/assets/hive-window-and-analytic-functions_select-result.webp deleted file mode 100644 index 60c472b68..000000000 Binary files a/topics/data/assets/hive-window-and-analytic-functions_select-result.webp and /dev/null differ diff --git a/topics/data/assets/hive-window-and-analytic-functions_windows.webp b/topics/data/assets/hive-window-and-analytic-functions_windows.webp deleted file mode 100644 index 42d717494..000000000 Binary files a/topics/data/assets/hive-window-and-analytic-functions_windows.webp and /dev/null differ diff --git a/topics/data/assets/introduction-to-apache-pig_problem-template.webp b/topics/data/assets/introduction-to-apache-pig_problem-template.webp deleted file mode 100644 index 207de0654..000000000 Binary files a/topics/data/assets/introduction-to-apache-pig_problem-template.webp and /dev/null differ diff --git a/topics/data/assets/introduction-to-crdt_b41416acdcc17fd248cefbd2f333fb6e_md5.webp b/topics/data/assets/introduction-to-crdt_b41416acdcc17fd248cefbd2f333fb6e_md5.webp deleted file mode 100644 index 4cf58bb78..000000000 Binary files a/topics/data/assets/introduction-to-crdt_b41416acdcc17fd248cefbd2f333fb6e_md5.webp and /dev/null differ diff --git a/topics/data/assets/local-first-software-20240706221930408.webp b/topics/data/assets/local-first-software-20240706221930408.webp deleted file mode 100644 index 3d2115dc0..000000000 Binary files a/topics/data/assets/local-first-software-20240706221930408.webp and /dev/null differ diff --git a/topics/data/assets/managed-table-vs-external-table_external-table.webp b/topics/data/assets/managed-table-vs-external-table_external-table.webp deleted file mode 100644 index 19b7a87d6..000000000 Binary files a/topics/data/assets/managed-table-vs-external-table_external-table.webp and /dev/null differ diff --git a/topics/data/assets/managed-table-vs-external-table_managed-table-check.webp b/topics/data/assets/managed-table-vs-external-table_managed-table-check.webp deleted file mode 100644 index f6f9d4405..000000000 Binary files a/topics/data/assets/managed-table-vs-external-table_managed-table-check.webp and /dev/null differ diff --git a/topics/data/assets/managed-table-vs-external-table_managed-table.webp b/topics/data/assets/managed-table-vs-external-table_managed-table.webp deleted file mode 100644 index 23c0f75ff..000000000 Binary files a/topics/data/assets/managed-table-vs-external-table_managed-table.webp and /dev/null differ diff --git a/topics/data/assets/managing-dataflow-and-sql-database-with-concurrency-control_c11c827159778f301fbd67b9d155b80b_md5.avif b/topics/data/assets/managing-dataflow-and-sql-database-with-concurrency-control_c11c827159778f301fbd67b9d155b80b_md5.avif deleted file mode 100644 index 3d8d9a520..000000000 Binary files a/topics/data/assets/managing-dataflow-and-sql-database-with-concurrency-control_c11c827159778f301fbd67b9d155b80b_md5.avif and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_combiner.webp b/topics/data/assets/mapreduce-components_combiner.webp deleted file mode 100644 index 9a858ca9b..000000000 Binary files a/topics/data/assets/mapreduce-components_combiner.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_distributed.webp b/topics/data/assets/mapreduce-components_distributed.webp deleted file mode 100644 index a563a61c1..000000000 Binary files a/topics/data/assets/mapreduce-components_distributed.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_input-splits-vs-blocks.webp b/topics/data/assets/mapreduce-components_input-splits-vs-blocks.webp deleted file mode 100644 index 1139665b3..000000000 Binary files a/topics/data/assets/mapreduce-components_input-splits-vs-blocks.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_map-phase.webp b/topics/data/assets/mapreduce-components_map-phase.webp deleted file mode 100644 index 5b29aa337..000000000 Binary files a/topics/data/assets/mapreduce-components_map-phase.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_multiple-reducers.webp b/topics/data/assets/mapreduce-components_multiple-reducers.webp deleted file mode 100644 index 24918173a..000000000 Binary files a/topics/data/assets/mapreduce-components_multiple-reducers.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_reduce-phase.webp b/topics/data/assets/mapreduce-components_reduce-phase.webp deleted file mode 100644 index 6680973e7..000000000 Binary files a/topics/data/assets/mapreduce-components_reduce-phase.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_stock-algorithm.webp b/topics/data/assets/mapreduce-components_stock-algorithm.webp deleted file mode 100644 index a70927f93..000000000 Binary files a/topics/data/assets/mapreduce-components_stock-algorithm.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce-components_stock-problem.webp b/topics/data/assets/mapreduce-components_stock-problem.webp deleted file mode 100644 index 7a8be7809..000000000 Binary files a/topics/data/assets/mapreduce-components_stock-problem.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce_mr-example-1.webp b/topics/data/assets/mapreduce_mr-example-1.webp deleted file mode 100644 index 219c62967..000000000 Binary files a/topics/data/assets/mapreduce_mr-example-1.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce_mr-example-2.webp b/topics/data/assets/mapreduce_mr-example-2.webp deleted file mode 100644 index d6e6e6a8e..000000000 Binary files a/topics/data/assets/mapreduce_mr-example-2.webp and /dev/null differ diff --git a/topics/data/assets/mapreduce_mr-example-3.webp b/topics/data/assets/mapreduce_mr-example-3.webp deleted file mode 100644 index 68f4fcaae..000000000 Binary files a/topics/data/assets/mapreduce_mr-example-3.webp and /dev/null differ diff --git a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-set-property.webp b/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-set-property.webp deleted file mode 100644 index 82be92c16..000000000 Binary files a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-set-property.webp and /dev/null differ diff --git a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-sort-by.webp b/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-sort-by.webp deleted file mode 100644 index b36337694..000000000 Binary files a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-sort-by.webp and /dev/null differ diff --git a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot.webp b/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot.webp deleted file mode 100644 index 3cab98153..000000000 Binary files a/topics/data/assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot.webp and /dev/null differ diff --git a/topics/data/assets/partitions-on-apache-hive_check-dynamic-partitions.webp b/topics/data/assets/partitions-on-apache-hive_check-dynamic-partitions.webp deleted file mode 100644 index 9320764df..000000000 Binary files a/topics/data/assets/partitions-on-apache-hive_check-dynamic-partitions.webp and /dev/null differ diff --git a/topics/data/assets/partitions-on-apache-hive_describe_stocks_partition.webp b/topics/data/assets/partitions-on-apache-hive_describe_stocks_partition.webp deleted file mode 100644 index cb5637087..000000000 Binary files a/topics/data/assets/partitions-on-apache-hive_describe_stocks_partition.webp and /dev/null differ diff --git a/topics/data/assets/partitions-on-apache-hive_inserting-b7j.webp b/topics/data/assets/partitions-on-apache-hive_inserting-b7j.webp deleted file mode 100644 index 2955005d7..000000000 Binary files a/topics/data/assets/partitions-on-apache-hive_inserting-b7j.webp and /dev/null differ diff --git a/topics/data/assets/partitions-on-apache-hive_location.webp b/topics/data/assets/partitions-on-apache-hive_location.webp deleted file mode 100644 index e5064eff9..000000000 Binary files a/topics/data/assets/partitions-on-apache-hive_location.webp and /dev/null differ diff --git a/topics/data/assets/partitions-on-apache-hive_show-partitions.webp b/topics/data/assets/partitions-on-apache-hive_show-partitions.webp deleted file mode 100644 index a5a467278..000000000 Binary files a/topics/data/assets/partitions-on-apache-hive_show-partitions.webp and /dev/null differ diff --git a/topics/data/assets/quick-learning-vector-database_ab8dc870fb19553f7616442fe4e9e6c5_md5.webp b/topics/data/assets/quick-learning-vector-database_ab8dc870fb19553f7616442fe4e9e6c5_md5.webp deleted file mode 100644 index e15bf3e3c..000000000 Binary files a/topics/data/assets/quick-learning-vector-database_ab8dc870fb19553f7616442fe4e9e6c5_md5.webp and /dev/null differ diff --git a/topics/data/assets/redis-leaderboard_flow_diagram.webp b/topics/data/assets/redis-leaderboard_flow_diagram.webp deleted file mode 100644 index 4509935ca..000000000 Binary files a/topics/data/assets/redis-leaderboard_flow_diagram.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-binary-search-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-binary-search-tree-example.webp deleted file mode 100644 index ea3fe7094..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-binary-search-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-complete-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-complete-binary-tree-example.webp deleted file mode 100644 index 60f6b571a..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-complete-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-full-strict-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-full-strict-binary-tree-example.webp deleted file mode 100644 index b5a058dac..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-full-strict-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-height-balanced-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-height-balanced-binary-tree-example.webp deleted file mode 100644 index a3b69bfc1..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-height-balanced-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-left-skewed-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-left-skewed-binary-tree-example.webp deleted file mode 100644 index 2f0bc75f7..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-left-skewed-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-perfect-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-perfect-binary-tree-example.webp deleted file mode 100644 index 0224862e6..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-perfect-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-right-skewed-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-right-skewed-binary-tree-example.webp deleted file mode 100644 index 6e3797582..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-right-skewed-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-node-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-node-example.webp deleted file mode 100644 index 1d045d9fc..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-node-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-file-system.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-file-system.webp deleted file mode 100644 index ef043a7d8..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-file-system.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-team.webp b/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-team.webp deleted file mode 100644 index 0e9706746..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-team.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_binary-tree-example.webp deleted file mode 100644 index 783331a87..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_height-of-a-node-in-a-bst.webp b/topics/data/assets/self-balanced-bsts---avl-trees_height-of-a-node-in-a-bst.webp deleted file mode 100644 index 4b3de5407..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_height-of-a-node-in-a-bst.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-1.webp b/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-1.webp deleted file mode 100644 index 78db90cc9..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-1.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-2.webp b/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-2.webp deleted file mode 100644 index 1ddbdc140..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-2.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-left.webp b/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-left.webp deleted file mode 100644 index 028e5a6b2..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-left.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-right.webp b/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-right.webp deleted file mode 100644 index 3fa5e0431..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-right.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_minimum-number-of-nodes-for-an-avl-tree-of-height-h.webp b/topics/data/assets/self-balanced-bsts---avl-trees_minimum-number-of-nodes-for-an-avl-tree-of-height-h.webp deleted file mode 100644 index 1af5f2947..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_minimum-number-of-nodes-for-an-avl-tree-of-height-h.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_non-binary-tree-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_non-binary-tree-example.webp deleted file mode 100644 index 36330e4e3..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_non-binary-tree-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-left.webp b/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-left.webp deleted file mode 100644 index 956c2e20a..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-left.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-right.webp b/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-right.webp deleted file mode 100644 index cf0d22f37..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-right.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_rotations-for-binary-tree-nodes.webp b/topics/data/assets/self-balanced-bsts---avl-trees_rotations-for-binary-tree-nodes.webp deleted file mode 100644 index 72e1a4cd9..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_rotations-for-binary-tree-nodes.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_runway-reservation-problem-example.webp b/topics/data/assets/self-balanced-bsts---avl-trees_runway-reservation-problem-example.webp deleted file mode 100644 index 3646673bb..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_runway-reservation-problem-example.webp and /dev/null differ diff --git a/topics/data/assets/self-balanced-bsts---avl-trees_the-best-way-to-build-a-tall-avl-tree-with-as-few-nodes-as-possible.webp b/topics/data/assets/self-balanced-bsts---avl-trees_the-best-way-to-build-a-tall-avl-tree-with-as-few-nodes-as-possible.webp deleted file mode 100644 index e2dfff7b9..000000000 Binary files a/topics/data/assets/self-balanced-bsts---avl-trees_the-best-way-to-build-a-tall-avl-tree-with-as-few-nodes-as-possible.webp and /dev/null differ diff --git a/topics/data/assets/sql-and-how-it-relates-to-disk-reads-and-writes_disk-sequential-random-access.webp b/topics/data/assets/sql-and-how-it-relates-to-disk-reads-and-writes_disk-sequential-random-access.webp deleted file mode 100644 index f938a8286..000000000 Binary files a/topics/data/assets/sql-and-how-it-relates-to-disk-reads-and-writes_disk-sequential-random-access.webp and /dev/null differ diff --git a/topics/data/base-model.md b/topics/data/base-model.md deleted file mode 100644 index 57f53c257..000000000 --- a/topics/data/base-model.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: Base model -description: The BASE model is a flexible NoSQL data approach focusing on Basic availability, Soft-state, and Eventual consistency to enable scalable and loosely consistent distributed databases. -date: 2022-04-18 -github_id: monotykamary -redirect: - - /NrrlVw ---- - -## What is the BASE model? - -BASE is an acronym for describing a flexible way to manipulate data. With requirements for NoSQL databases, the BASE model came to be as a less pessimistic approach of the [[ACID model]] when handling data. The acronym stands for: - -- **Basic availability:** The database is available and should work most of the time. -- **Soft-state:** Data stores don't have to be completely write-consistent or require mutual consistency between replicas. -- **Eventual consistency**: The database is lazily consistent, meaning it ensures data consistency at a later point. - -![](assets/base-model_base_model_diagram.webp) - -BASE properties are significantly looser than ACID, but the tradeoff allows for scalability. Although BASE has loose consistency, it doesn't mean data will be completely inconsistent. However, it does require assistance from the developer when ensuring what the data should focus on being consistent in (e.g: consistent in time of activity or order of processing). - -The BASE model loosely refers to distributed data stores, which means these databases require different approaches when handling transactions and events asynchronously. This will also mean a tradeoff of availability for consistency in the presence of a network partition or power outage; this is more deeply covered in [[CAP theorem]]. - -#### Reference - -- https://phoenixnap.com/kb/acid-vs-base -- https://neo4j.com/blog/acid-vs-base-consistency-models-explained/ diff --git a/topics/data/behind-a-hive-table.md b/topics/data/behind-a-hive-table.md deleted file mode 100644 index 02533019a..000000000 --- a/topics/data/behind-a-hive-table.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Behind a hive table -description: Learn how to create and understand Hive tables, including table structure, storage formats, SerDe functions, HDFS data location, and metadata management in the Hive metastore. -date: 2022-11-11 -github_id: dudaka -icy: 10 -redirect: - - /qAShRQ ---- - -From the last article, [[Introduction to Apache Hive]], we saw how to create a database and a table in Hive. In this article, we will look at the details behind a Hive table. More importantly, we'll see what are the essential elements that is needed behind the Hive table. We've already created this database `stocks_db` in our last article, so we won't do it again. Thus, we will execute `USE stocks_db;` to switch to the database `stocks_db`. Let's now create a table for the stocks data set. - -```sql -CREATE TABLE IF NOT EXISTS stocks ( -exch string, -symbol string, -ymd string, -price_open float, -price_high float, -price_low float, -price_close float, -volume int, -price_adj_close float) -ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; -``` - -The table name is `stocks`, we have listed all the columns in our stocks data set. Our data set is a comma delimited data set, so that is why in the create table instruction we will say `ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';` - -The `DESCRIBE FORMATTED stocks;` command will show the detailed information about the stocks table. This information is very useful especially if the table is not created by ourselves but we want to know more information about the table. - -![](assets/behind-a-hive-table_describe-formtted-stocks.webp) - -First, it is the information about the columns in the table and its data types, which are very similar to Java data types. Next is the information about the database that the table belongs to create a timestamp, etc. Now, let's look at two important attributes of the table, the first one is the location and the second one is the table type. The location attribute specifies the HDFS location from where the table will look up the data set. In fact, when creating a table, we can specify the location attribute if the data set already exists in our Hadoop cluster. So what happens when we don't specify the location attribute? By default, whenever we create a database Hive creates a directory in HDFS and whenever we create a table, by default, Hive creates a directory for the table under the database directory in HDFS. As shown in the above screenshot, with our `stocks_db` database, Hive has created a directory in HDFS named `stocks_db.db` and for the table stocks Hive has created a directory in HDFS named `stocks` under the `stocks_db.db` directory in HDFS. Both these directories are created under `user/hive/warehouse` directory. It is also simply referred to as the Hive Warehouse directory. Next important attribute is the table type. There are two types of tables in Hive: the first type is called MANAGED_TABLE and the second type is called EXTERNAL_TABLE. We're going to look at table types in detail in a separate article. Let's move on to the next set of attributes under `storage information`. These attributes will tell us about the format of the data set and how the data is read and written during Hive execution. When we don't specify a table format when creating a table, Hive will make TextInputFormat to be the default input format and HiveIgnoreKeyTextOutputFormat as the default output format. HiveIgnoreKeyTextOutputFormat is very similar to TextOutputFormat the only difference is the value of the key is ignored in the output. But what is SerDe? How is SerDe different from the input and output format? SerDe stands for serializer and deserializer. When reading the data, deserializer from SerDe is involved and when writing the data serializer from SerDe is involved. - -![](assets/behind-a-hive-table_serde-reading-data.webp) - -Let’s assume that we want to find the maximum volume for a stock symbol like GE, for instance, from our stocks table shown in the above diagram. We will write a query `SELECT max(volume) FROM stocks WHERE symbol = ‘GE’`. At runtime, Hive should be able to parse the record, find the column value that corresponds to volume and map the value to the volume column in the table. Hive knows our input is a text file since the input format is TextInputFormat. Thus, the record reader implementation in the input format will return line by line as records from the data set. Once Hive has a record, which is nothing but a line of text, the deserialized method in SerDe deserializes the record as a whole. Now the entire record is deserialized and now we need some help to get the needed data from the record. In our case, we just need the volume column. Mapping the deserialized record into columns is done by a class called object inspector in SerDe. The object inspector knows how to construct individual fields. Again, in our use-case, the object inspector will know how to extract the column value out of a deserialized record. - -![](assets/behind-a-hive-table_serde-writing-data.webp) - -The exact reverse happens during a write operation, that is, when we’re trying to insert something into a table, serialize method from SerDe will be given a deserialized object representing a record. The serialized method will make use of the object inspector to get the individual fields in the record and convert the record to the appropriate type mapping back to the table. The output format then writes the record into HDFS. That is how SerDe input and output format work together. - -Let's go back to the details of the table, the screenshot, SerDe is simply a Java class. When we don't specify a SerDe, the default SerDe is LazySimpleSerDe. We can also write custom SerDe as well. SerDe can be very powerful. Data may have several columns and is represented in JSON format and the data types of each column is not necessarily a simple type like string or float. Meaning most of the columns are complex types like structures arrays and they are nested. In such instance, SerDe helps in two ways: first, to interpret complex and nested data and map it back to our columns in the table and second, to map just the needed columns instead of mapping all the columns from the data set. We'll see that in more detail when we talk about collection types in Hive in another article. - -Now we know about SerDe, we've also created a table and know the details and the meaning of important attributes of the table. But what really happens when creating a table, meaning we know where the data for the table is stored in HDFS, but where the metadata for the table is stored? In other words, where is the information in the screenshot stored? The metadata of the table has to be persisted somewhere such that it can be looked up outside of this existing Hive session. When creating a Hive table the metadata of the table is stored in a database like MySQL or Oracle and it is called the metastore. Usually, MySQL is used in production setups. The metastore will only store the metadata information and not the data itself. The data itself will reside in HDFS. But how does Hive know where the database in MySQL resides? And how to connect to it? This information along with several other key properties are stored in the `hive-site.xml` file. The location of the file would vary from cluster to Cluster, for example, `hive-site.xml` can be found under the `/etc/hive/conf` directory. - -![](assets/behind-a-hive-table_hive-site.xml.webp) - -Here is the contents of `hive-site.xml`, the very first property is the connection URL property will list the URL for MySQL, where the metastore database will be stored. This file also has information about the JDBC driver to be used and other connection properties like username and password to connect to the database. Therefore, now reading this file, Hive knows exactly how to connect to the metastore database, where just the metadata for the hive tables and databases are stored. - -Now we know how to create a table and where the metadata for the table resides. Let's finish this article by looking at a few drop commands. We can drop the database with drop database command, i.e. `DROP DATABASE stocks_db` and can drop the table with drop table command, i.e. `DROP TABLE stocks`. However, we can drop a database only when the database is empty. If the database already has tables, we will either have to drop the tables first before attempting to drop the database or do a cascade drop with command `DROP DATABASE CASCADE;`. diff --git a/topics/data/bloom-filter.md b/topics/data/bloom-filter.md deleted file mode 100644 index dfe75bfd7..000000000 --- a/topics/data/bloom-filter.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: Bloom filter -description: "A Bloom filter is a probabilistic data structure used for testing whether an element is a member of a set or not. It's space-efficient compared to other data structures like hash tables, but it may give false positives (indicating that an element is in the set when it's not) and never gives false negatives (indicating that an element is not in the set when it actually is not)..." -date: 2024-06-28 -authors: - - nnhuyhoang -tags: - - big-o - - bloom-filter - - data-structure -redirect: - - /RQEpfsM ---- - -## What is Bloom filter? - -A Bloom filter is a probabilistic data structure used for testing whether an element is a member of a set or not. It's space-efficient compared to other data structures like hash tables, but it may give false positives (indicating that an element is in the set when it's not) and never gives false negatives (indicating that an element is not in the set when it actually is not). - -![](assets/bloom-filter_bloom_filter.webp) - -## How Bloom filter work? - -`Initialization`: You start with a bit array of size `m` (usually a large prime number) initially set to all zeros, and `k` different hash functions. - -`Adding Elements`: When you want to add an element to the Bloom filter, you run it through each of the `k` hash functions, each of which maps the element to one of the `m` bits in the array. You then set those bits to `1`. - -`Checking Membership`: To check if an element is in the Bloom filter, you again run it through each of the `k` hash functions. If all `k` bits are set to `1` in the bit array, the element is probably in the set. If any of the bits are `0`, then the element is definitely not in the set. However, due to potential hash collisions, `false positives are possible`. - -## When to use? - -Imagine that you have a system which millions user, and then you have a user table which had millions of records. So everytime a new user is registered, you have to query millions records table to check username is existed or not which cost `Time complexity = O(n)` (`n` will be vary upto millions) in case not aggregate further. Instead of that, you can `Checking membership` in Bloom filter which just have `Time complexity = O(k)` (k is constant number of hash function) and then response to user that username is alreay existed. The result has a small probability of false positive(username not existed but response existed) due to potential hash collion but it acceptable by notify user can register with other username. - -Besides above scenario, Bloom filter can apply in other case like: - -- `Caching Systems`: Bloom Filters are commonly used in caching systems to quickly check if a requested item is likely to be in the cache before performing a more expensive lookup. This helps in reducing cache misses and improving overall system performance. - -- `Data Deduplication`: In systems where duplicate data needs to be identified and eliminated efficiently, Bloom Filters can help quickly determine whether a new data item is a duplicate of an existing item in the dataset. - -- `Web Crawlers and Search Engines`: Bloom Filters can be used in web crawlers and search engines to quickly check whether a URL has been visited or indexed before, reducing redundant crawling or indexing efforts. - -- `Spell Checkers`: Bloom Filters can be employed in spell checkers to quickly determine if a word is misspelled by checking against a large dictionary of correctly spelled words. - -- `Network Routing`: In networking applications, Bloom Filters can be used to quickly filter out packets or messages that are not likely to match certain criteria, reducing the computational load on routers and improving network performance. - -- `Blockchain and Distributed Systems`: In distributed systems like blockchain networks, Bloom Filters can be utilized for lightweight and efficient synchronization of data among nodes, reducing the amount of data that needs to be transmitted. - -Overall, Bloom Filters are suitable for applications where memory usage needs to be minimized, and a small probability of false positives is acceptable. However, they are not suitable for scenarios where false positives are unacceptable or where exact membership information is required. - -## Pros and Cons? - -### Pros: - -- `Space Efficiency`: Bloom Filters are very space-efficient compared to other data structures like hash tables or trees. They require only a fraction of the space that would be needed to store the actual set of elements. - -- `Fast Membership Testing`: Bloom Filters provide constant-time (O(k)) complexity for membership testing, regardless of the size of the set. This makes them suitable for large datasets where quick membership checks are required. - -- `No False Negatives`: Bloom Filters never produce false negatives. If the filter says an element is not in the set, it's guaranteed to be absent. This property is useful in applications where false negatives are unacceptable. - -- `Versatility`: Bloom Filters are versatile and can be used in various applications such as caching, spell checking, network routing, and more. They are particularly useful in scenarios where approximate membership testing is acceptable. - -### Cons: - -- `False Positives`: Bloom Filters can produce false positives, indicating that an element is in the set when it's not. The probability of false positives increases with the number of elements in the set and the parameters chosen for the Bloom filter (e.g., size of the bit array and number of hash functions). - -- `Cannot Remove Elements`: Bloom Filters do not support element removal. Once an element is added to the filter, it cannot be removed without resorting to complex techniques or resetting the filter entirely. - -- `Parameter Sensitivity`: The performance and effectiveness of a Bloom filter are highly sensitive to the parameters chosen, such as the size of the bit array (m) and the number of hash functions (k). Selecting inappropriate parameters can lead to higher false positive rates or increased memory usage. - -- `Limited Operations`: Bloom Filters support only two primary operations: adding elements and checking for membership. They do not support other common operations like element retrieval or iteration over the elements in the set. - -- `Hash Function Dependency`: Bloom Filters rely heavily on hash functions for their operation. The quality of the hash functions used can significantly impact the performance and effectiveness of the Bloom filter. - -## Simple Implementation: - -Nowadays, many cache system integrated bloom filter as a feature. Can use it without implementation, but if you want try to hand-on. the simple implementation is a example: - -```go -package main - -import ( - "fmt" - "hash/crc32" - "hash/fnv" -) - -type BloomFilter struct { - bitArray []bool - size uint - hashFunc []func(data string) uint -} - -func NewBloomFilter(size uint) *BloomFilter { - bf := &BloomFilter{ - bitArray: make([]bool, size), - size: size, - hashFunc: []func(data string) uint{ - func(data string) uint { return uint(fnv.New32().Sum32()) }, - func(data string) uint { return uint(crc32.ChecksumIEEE([]byte(data))) }, - // Add more hash functions here if needed - }, - } - return bf -} - -func (bf *BloomFilter) Add(data string, numHash int) error { - if numHash > len(bf.hashFunc) { - return fmt.Errorf("number of hash functions exceeds the limit") - } - for i := 0; i < numHash; i++ { - index := bf.hashFunc[i](data) % bf.size - bf.bitArray[index] = true - } - return nil -} - -func (bf *BloomFilter) Contains(data string) bool { - for _, hashFunc := range bf.hashFunc { - index := hashFunc(data) % bf.size - if !bf.bitArray[index] { - return false - } - } - return true -} - -func main() { - bloomFilter := NewBloomFilter(100) - - // Add user identities to the Bloom filter - userIdentities := []string{"user123", "user456", "user789"} - for _, identity := range userIdentities { - bloomFilter.Add(identity, 2) - } - - // Check for existence of user identities - fmt.Println(bloomFilter.Contains("user123")) // true - fmt.Println(bloomFilter.Contains("user789")) // true - fmt.Println(bloomFilter.Contains("user999")) // false -} -``` diff --git a/topics/data/buckets-on-apache-hive.md b/topics/data/buckets-on-apache-hive.md deleted file mode 100644 index 1e271e19f..000000000 --- a/topics/data/buckets-on-apache-hive.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: Buckets on Apache Hive -description: Learn how Apache Hive buckets solve issues with dynamic partitions by reducing tiny files, enabling efficient sampling, and improving query performance through bucketing and partitioning techniques. -date: 2022-12-10 -github_id: dudaka -icy: 10 -redirect: - - /fMaFEw ---- - -With the understanding about partitions, the benefits of partitions and how to work with partitions from the article [[Partitions on Apache Hive]], we are going to see a couple of potential problems that we may see with partitions, especially, with dynamic partitions. And, of course, how to address them using buckets. In this article, we'll discuss about: - -- What are buckets? -- The differences between buckets and partitions. -- The benefits of using buckets in Hive. -- Creating bucketed tables in Hive and work with it. - -Assume that we created a partition table named `stocks_dynamic_partition` which is partitioned by three columns: exchange name `exch_name`, year `yr` and symbol `sym`. Thus, we dynamically created hundreds of partitions on the table. Let's execute `SHOW PARTITIONS stocks_dynamic_partition;` to list all the partitions from the table. - -![](assets/buckets-on-apache-hive_listofpartitions.webp) - -The screenshot shows that this table has about 362 partitions. We see a lot of partitions a lot of partitions for just 2003 and similarly for 2002 and similarly for 2001. Let's pick a symbol and go into the directory for a specific year. - -![](assets/buckets-on-apache-hive_lsapartition.webp) - -As shown in the screenshot, there is a tiny file under the partition symbol `BUB` under year 2003. -There are two problems here: - -- Problem 1: too many partitions. More symbol for a given year will end up with that many partitions. Meaning that for each year, based on the number of new symbols added to the exchange that year, the number of partitions can actually vary and it is not predictable. -- Problem 2: tiny files under the partitions. We know that Hadoop is not an ideal platform to deal with tiny files. We may argue that symbol is not the best column to partition a data byte but nevertheless, we could be facing a similar scenario in our job. - -So, how do we address these two problems? The answer is buckets. Let's consider the following create table statement for creating a table `stocks_bucket`: - -```sql -CREATE TABLE IF NOT EXISTS stocks_bucket ( -exch STRING, -symbol STRING, -ymd STRING, -price_open FLOAT, -price_high FLOAT, -price_low FLOAT, -price_close FLOAT, -volume INT, -price_adj_close FLOAT) -PARTITIONED BY (exch_name STRING, yr STRING) -CLUSTERED BY (symbol) INTO 5 BUCKETS -ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; -``` - -In this table, we are partitioning the table by exchange name and year, then we are saying `CLUSTERED BY (symbol) INTO 5 BUCKETS` which is that in first partition, the data is set by exchange name and year and once the data set is partitioned by year, the records for the year are stored into five buckets. In other words, five files using the symbol as our bucketing column. Each symbol is assigned to a bucket number using a hash function and all the records for that symbol will be stored into the assigned bucket. For example, if symbol `XYZ` is assigned bucket number 3, all records for `XYZ` will be stored in bucket number 3. - -![](assets/buckets-on-apache-hive_bucketingdemo.webp) - -If we execute the describe command on the table as shown in the screenshot, and we can see the table is partitioned with two columns: exchange name and year, and the bucketed column symbol and the number of buckets is set to 5. Let's insert records into this table. - -```sql -INSERT OVERWRITE TABLE stocks_bucket -PARTITION (exch_name='ABCSE', yr) -SELECT *, year(ymd) -FROM stocks WHERE year(ymd) IN ('2001', '2002', '2003') and symbol like 'B%'; -``` - -The insert is just like any other insert, but make sure that `hive.enforce.bucketing` equals `true` (`SET hive.enforce.bucketing = true;`). Since this insert is going to create dynamic partitions as well, all the properties needed for dynamic partitions have to be set: - -``` -SET hive.exec.dynamic.partition=true; - -SET hive.exec.max.dynamic.partitions=1000; - -SET hive.exec.max.dynamic.partitions.pernode=500; - -SET hive.enforce.bucketing = true; -``` - -![](assets/buckets-on-apache-hive_insertdata.webp) - -![](assets/buckets-on-apache-hive_insertdone.webp) - -As shown in the screenshot, the number of reduced tasks determined at compile time equals 5 which equals to the number of buckets on this table which is also 5. When the insert is complete, three partitions got created for this table and under each partition, there are five buckets, so the number of files is 15 (five buckets under each partition). Let's look at the partition for year 2002. - -![](assets/buckets-on-apache-hive_lsbucketing.webp) - -As shown in the above screnshot, there are only five files or five buckets as opposed to too many tiny partitions. This is the benefit of buckets, we will get a constant number of buckets and also avoid tiny files. The second benefit of buckets is `sampling`. Sampling is beneficial when we don't want to query the entire data set and we would only like to analyze a random sample. - -```sql ---Table sampling with out buckets -hive> SELECT * -FROM stocks TABLESAMPLE(BUCKET 3 OUT OF 5 ON symbol) s; - ---Table sampling with buckets -hive> SELECT * -FROM stocks_bucket TABLESAMPLE(BUCKET 3 OUT OF 5 ON symbol) s; -``` - -In the first select, we're doing a table sample on a table `stocks` which is not bucketed and asking for bucket 3 out of 5 buckets based on the column symbol. Since this table is not bucketed, Hive has to randomly assign symbols into five buckets and rows which belong to the third bucket will be returned. The problem with this query is that: to return bucket number 3, the table sample needs to scan the entire table because the table is not bucketed and this is time intensive. On the other hand, the second select on the bucketized `stocks_buckets` table is efficient than the first one as the table we are sampling is bucketized and also the sampling is done on the bucketized column `symbol`. Hence, this query will be more efficient than the first one. The other benefits of buckets is its efficiency during map side joints. We'll look more detail into that in other article about optimizations. - -In summary, we now understood what buckets are. We saw the difference between buckets and partitions. And we also know how to work with buckets. There are three benefits of buckets: (1) unlike partitions the number of buckets is constant and solves the tiny files issue, (2) buckets are very efficient when sampling tables and (3) finally the benefit of using bucket is during map site joints which we'll discuss in more detail later. diff --git a/topics/data/cap-theorem.md b/topics/data/cap-theorem.md deleted file mode 100644 index 2e748dd46..000000000 --- a/topics/data/cap-theorem.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Cap theorem -description: Learn about the CAP theorem, which explains how distributed databases balance consistency, availability, and partition tolerance, and how different database types prioritize these guarantees. -date: 2022-04-18 -github_id: monotykamary -redirect: - - /TbxCnQ ---- - -## What is CAP theorem - -CAP theorem, or also named Brewer's theorem (from scientist Eric Brewer), states that any distributed data store can provide only two of three guarantees: - -- **Consistency:** Any read operations are up-to-date with the latest write operation, meaning that all clients see the same data at a given point of time. -- **Availability:** In a distributed data store, any and all working nodes in the system should return a valid response for any request, without exception. -- **Partition tolerance**: This refers to tolerance for an event that causes a network or power disturbance between 2 or more nodes. Tolerance here means that the cluster should continue to work despite drops in communication between nodes. - -![](assets/cap-theorem_cap_theorem_diagram.webp) - -## CAP theorem database types - -- **CP database:** A CP database sacrifices availability for consistency and partition tolerance. In a network partition, non-consistent nodes are shut down until the partition is resolved. -- **AP database:** An AP database delivers availability and partition tolerance at the cost of consistency. In a network partition, read consistencies occur between nodes, such that clients accessing one node may see different data than other clients. This inconsistency occurs until the partition is resolved, in which case the system will try to re-sync all nodes to repair all inconsistencies. -- **CA database:** A CA database achieves consistency and availability across all nodes. In a network partition, request for data will not return a valid response, as any partition will mean a loss of consistency. As such, such a database is not fault-tolerant. - -## In reality - -Eric Brewer makes it a point that you can only have 2 out of 3 guarantees in CAP theorem, which is not completely true. You can ensure at least a subset of guarantees between all three parts of the CAP theorem. There will always be nuanced tradeoffs in such a rounded approach to a distributed data store. - -#### Reference - -- https://en.wikipedia.org/wiki/CAP_theorem -- https://www.ibm.com/cloud/learn/cap-theorem diff --git a/topics/data/data-vault-modelling.md b/topics/data/data-vault-modelling.md deleted file mode 100644 index 99667f99b..000000000 --- a/topics/data/data-vault-modelling.md +++ /dev/null @@ -1,178 +0,0 @@ ---- -title: Data vault modelling -description: Learn how Data Vault architecture builds flexible, scalable data lakes using hubs, links, and satellites to manage business keys and historical data with pros and cons explained. -date: 2022-12-15 -github_id: Bachpx2911 -icy: 10 -redirect: - - /s9Fo7w ---- - -## Introduction - -When looking to build out a new data lake, one of the most important factors is to establish the warehousing architecture that will be used as the foundation for the -data platform. While there are several traditional methodologies to consider when establishing a new data lake (from Inmon and Kimball, for example), one alternative -presents a unique opportunity: Data Vault. In this article, I will show basic knowledge of data vault and some pros and cons of it. - -## Overview of data vault - -Data Vault is created by Dan Linstedt and his team at Lockheed Martin in the early 90s. It includes 3 main cores: Hub, Link and Satellite. Hub represents a core business concept, such as they represent Customer Id/Product Number/Vehicle identification number (VIN). Users will use a business key to get information about a Hub. Hubs don’t contain any context data or details about the entity. They only contain the defined business key and a few mandated Data Vault fields. A critical attribute of a Hub is that they contain only one row per key. - -Meanwhile, A Link defines the relationship between business keys from two or more Hubs. Just like the Hub, a Link structure contains no contextual information about the entities. There should also be only one row representing the relationship between two entities. - -In order to represent a relationship that no longer exists, we would need to create a satellite table off this Link table which would contain an is_deleted flag; this is known as an Effectivity Satellite. - -## Example of data vault - -We can have an example of data vault model here. For example, we have relationship database as example: - -Employee table: - -``` -employee_id(PK), -job_id(FK), -department_id(FK), -hire_Date, -salary, -first_name, -last_name, -manager_id -``` - -Job table: - -``` -job_id(PK), -job_name, -min_salary, -max_salary -``` - -Department table: - -``` -department_id(PK), -department_name, -location_id -``` - -Location table: - -``` -location_id(PK), -street, -city, -district -``` - -From these tables, we will create hub,link and sattelite: - -Hub_employee: - -``` -HK_employee(hash key of employee_id), -employee_id, -load_dts, -source -``` - -Hub_job: - -``` -HK_job(hash key of job_id), -job_id, -load_dts, -source -``` - -Hub_department: - -``` -HK_department(hash key of department_id), -department_id, -load_dts, -source -``` - -Hub_location: - -``` -HK_location(hash key of location_id), -location_id, -load_dts, -source -``` - -Link_employee: - -``` -Emp_Job_Dep_HK(hash key of employee_id, job_id, department_id), -employee_id, -job_id, -department_id, -load_dts, -source -``` - -Link_department: - -``` -Dep_Loc_HK(hash key of department_id, location_id), -department_id, -location_id, -load_dts, -source -``` - -HAL_link: - -``` -HAL_HK(hash key of manager_id,employee_id), -manager_id, -employee_id, -load_Dts, -source -``` - -Sat_employee: - -``` -HK_employee(hash key of employee_id), -first_name, -last_name, -email, -phone_number -``` - -Sat_location: - -``` -HK_location(hash key of location_id), -street,city, -district -``` - -Each hub will represent for entity in database. The primary key of each table is hask key of business key. Then we will create dimension and fact table through join -between link, hub and sat table. - -## Pros and cons of data vault - -Pros: flexibility, maintainability and scalability both in terms of semantic complexity and sheer volume. - -It aims to facilitate the above by introducing three major design principles that set it apart from an EDW based on 3NF or on dimensional modelling: - -- It decouples management of business keys from any attributes of business entities (Hubs/Links and their Satellites). -- It expects any relationship between business entities within the system to be modelled as many-to-many (a link table is introduced for any such relationship). -- It assumes that all attributes are maintained in a way, similar to type 2 slowly changing dimension of Kimball-style dimensional modelling. - -These design principles allow the system to use MPP for any ETL processes (all entities can be loaded at the same time), facilitate Master Data Management (it offers great flexibility in adding/removing data sources) and provides a robust framework for recording historical data. - -Cons: - -- Two to three-fold explosion of the number of tables compared to 3NF modelling -- Data Vault EDW does not focus on read performance: large number of relationships between tables and complexity of joins often require separate layer of bridging tables to be maintained as materialised views of the data vault - -## References - -- https://www.phdata.io/blog/building-modern-data-platform-with-data-vault/ -- https://www.databricks.com/glossary/data-vault diff --git a/topics/data/dbt-the-good-solution-to-accelerate-data-transformation.md b/topics/data/dbt-the-good-solution-to-accelerate-data-transformation.md deleted file mode 100644 index 3f8393dbf..000000000 --- a/topics/data/dbt-the-good-solution-to-accelerate-data-transformation.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: Dbt the good solution to accelerate data transformation -description: Discover how DBT (data build tool) simplifies data transformation in warehouses with modular SQL, automation, and software engineering practices for faster, reliable analytics and trusted data delivery. -date: 2022-11-21 -github_id: Bachpx2911 -icy: 10 -redirect: - - /DG5yGw ---- - -Transformation is one of the most important process in building data warehouse. It will allow you to clean, combine, remove duplicates, reorganize, and filter all your data. The transformation will enable your enterprise to develop useful and reliable insights via analytics and There are several tools on market for it such as Striim, Pentaho, Hevo Data..Yet, the one that clearly stands out, in particular, the data build tool. This article will introduce and review DBT in transforming data. -According to DBT documentation, the tool is a development framework that combines modular SQL with software engineering best practices to make data transformation reliable, fast, and fun. It makes data engineering activities accessible to people with data analyst skills to transform the data in the warehouse using simple select statements, effectively creating your entire transformation process with code. You can write custom business logic using SQL, automate data quality testing, deploy the code, and deliver trusted data with data documentation side-by-side with the code. In short, DBT (data build tool) turns your data analysts into engineers and allows them to own the entire analytics engineering workflow. - -So what make DBT more powerful than other tools? There are some advantage when Data Engineer or Data Analyst using DBT: - -1. DBT is an open-source application written in Python, giving the users the power to customize it as needed. By using Jinja macro and SQL, your code in DBT will be simple - and short. We can see it in below example: - Normal SQL - -```sql -select -order_id, -sum(case when payment_method = 'bank_transfer' then amount end) as bank_transfer_amount, -sum(case when payment_method = 'credit_card' then amount end) as credit_card_amount, -sum(case when payment_method = 'gift_card' then amount end) as gift_card_amount, -sum(amount) as total_amount -from {{ ref('raw_payments') }} -group by 1 - -``` - -SQL with Jinja - -```select -order_id, -{% for payment_method in ["bank_transfer", "credit_card", "gift_card"] %} -sum(case when payment_method = '{{payment_method}}' then amount end) as {{payment_method}}_amount, -{% endfor %} -sum(amount) as total_amount -from {{ ref('raw_payments') }} -group by 1 -``` - -2. It also offers a lot of flexibility to the users. Say, for example, the resultant project structure is not a match for your organizational needs. You can customize it by - editing the dbt_project.yml file or the configuration file and rearranging the folders. - -3. It Apply software engineering practices—such as modular code, version control, testing, and continuous integration/continuous deployment (CI/CD)—to analytics code. - Controlling code version may be useful to manage logic in building data warehouse - -4. Data documentation is accessible, easily updated, and allows you to deliver trusted data across the organization. DBT automatically generates documentation around descriptions, models dependencies, model SQL, sources, and tests. DBT creates lineage graphs of the data pipeline, providing transparency and visibility into what the data is describing, how it was produced, as well as how it maps to business logic. - -Beside these features, DBT also provide other features as other transformation tool. However, it also have some cons. Firstly, you will still need an additional tool or -tools to do the extract and load steps to carry out the process since the Data Build Tool only handles the T aspect of ETL. Secondly, due to DBT it is SQL-based, it provides less readability than tools with an interactive UI. And lastly, if you want to transform with database statement such as merge loop model, DBT may not support this. - -In conclusion, I think DBT is a good tool for transforming data. I think that this tool will continue developing more and becoming popular in the future - -## References - -- https://www.analytics8.com/blog/dbt-overview-what-is-dbt-and-what-can-it-do-for-my-data-pipeline/ -- https://docs.getdbt.com/docs/get-started/learning-more/using-jinja diff --git a/topics/data/full-text-search-with-postgresql.md b/topics/data/full-text-search-with-postgresql.md deleted file mode 100644 index 81160d6e5..000000000 --- a/topics/data/full-text-search-with-postgresql.md +++ /dev/null @@ -1,239 +0,0 @@ ---- -title: Full text search with postgresql -description: Learn how to boost PostgreSQL search performance using full-text search with GIN indexes, stop words, and custom configurations for faster and flexible text queries on large datasets. -date: 2022-07-12 -github_id: nnhuyhoang -redirect: - - /7UoOww ---- - -## What is Full-text search? - -Full-Text Search refers to a technique in which you search for a single computer-stored document or a collection in your full-text database. It provides you with the capability to identify natural-language documents that satisfy a query. - -## Why need Full-text search? - -Normally when we want to search for some words or text in a long sentence, we usually use `LIKE` operator. - -```sql -SELECT * FROM tweets WHERE content ILIKE '%something%'; -``` - -However, using `LIKE` operator with the leading wildcard will make PostgreSQL perform `Seq Scan` which means database will skip using index for finding matched records. It will cause very low performance for big data tables. Therefore, it's where `FTS` can be used to boost the performance in queries. - -## Indexing - -For normal columns, using [B-Tree index](https://dzone.com/articles/database-btree-indexing-in-sqlite) is the most common selection. However, in `FTS`, we should apply [GIN index](https://www.postgresql.org/docs/11/gin-intro.html) (**Generalized Inverted**). `GIN index` was designed to deal with data types that are subdividable and you want to search for individual component values (array elements, lexemes in a text document, etc). For simple explanation, `GIN index` like the table of contents in a book, where the heap pointers (to the actual table) are the page numbers. Multiple entries can be combined to yield a specific result. - -## Stop words - -For most human languages, there are some words that do not have any value in searching or analyzing, those words are called stop-words. For example, In English, some stop-words can be: `is, the, and, in, so,... etc`. Therefore, when we filter text, it should not count in our search string. - -## Hands-on - -Following are the step-by-step instruction to implement Full-text search in PostgresSQL: - -#### 1. Create `GIN INDEX` for `vector` column: - -```sql -CREATE INDEX idx_vector ON tweets USING GIN(vector); -``` - -#### 2. Insert data into `vector` column - -```sql -UPDATE - tweets t - SET - vector = array_to_tsvector (( - SELECT - array_agg(DISTINCT substring(lexeme FOR len)) - FROM - unnest(to_tsvector(LOWER(t."content"))), - generate_series(1, length(lexeme)) len)); -``` - -To explore what is exactly the above query does? Let's split it into smaller parts to better explain: - -**Convert sentence to vector** - -```sql -SELECT to_tsvector(LOWER('.@TataSky on which channel #WorldCupFinal #football is showing which ever is being tuned its paid channel.')); -``` - -It will return a vector where every token is a lexeme (a unit of lexical meaning) with its position in the sentence. - -```sql -'channel':4,16 'ever':10 'footbal':6 'paid':15 'show':8 'tataski':1 'tune':13 'worldcupfin':5 -``` - -**Converting vector into a table-like structure** - -```sql -SELECT * FROM unnest(to_tsvector(LOWER('.@TataSky on which channel #WorldCupFinal #football is showing which ever is being tuned its paid channel.'))); -``` - -It will return a table-like structure for above vector: - -| lexeme | positions | weights | -| ----------- | --------- | ------- | -| channel | {4,16} | {D,D} | -| ever | {10} | {D} | -| footbal | {6} | {D} | -| paid | {15} | {D} | -| show | {8} | {D} | -| tataski | {1} | {D} | -| tune | {13} | {D} | -| worldcupfin | {5} | {D} | - -**Get every substring for each lexeme** - -```sql -SELECT - DISTINCT substring(lexeme FOR len) -FROM - unnest(to_tsvector(LOWER('.@TataSky on which channel #WorldCupFinal #football is showing which ever is being tuned its paid channel.'))), - generate_series(1, length(lexeme)) len; -``` - -It will return every distinct substring for every lexeme in the vector: - -```sql -ever -tatask -tun -worldcupf -wor -ta -worldcupfin -tatas -tu -ch -pa -ev -tat -wo -footbal -worldcup -foo -worldcu -channe -chann -c -eve -cha -tata -paid -tune -tataski -e -channel -sho -footb -s -w -worldcupfi -pai -sh -chan -show -worldc -worl -world -f -foot -fo -p -t -footba -``` - -Therefore, in the end, every vector column record will have a value like the below: - -```sql -'c' 'ch' 'cha' 'chan' 'chann' 'channe' 'channel' 'e' 'ev' 'eve' 'ever' 'f' 'fo' 'foo' 'foot' 'footb' 'footba' 'footbal' 'p' 'pa' 'pai' 'paid' 's' 'sh' 'sho' 'show' 't' 'ta' 'tat' 'tata' 'tatas' 'tatask' 'tataski' 'tu' 'tun' 'tune' 'w' 'wo' 'wor' 'worl' 'world' 'worldc' 'worldcu' 'worldcup' 'worldcupf' 'worldcupfi' 'worldcupfin' -``` - -#### 3. Query to find text (it will use default English stop-words) - -```sql -SELECT * FROM tweets WHERE vector @@ to_tsquery(REPLACE(LOWER('multiple words with no order'),' ', ' & ')); -``` - -#### 4. Custom Search configuration (OPTIONAL) - -We can also configure the search configuration on our own like a custom stop-words template: - -```sql -CREATE TEXT SEARCH DICTIONARY english_stem_nostop ( - Template = snowball - , Language = english -); - -CREATE TEXT SEARCH CONFIGURATION public.english_nostop ( COPY = pg_catalog.english ); - -ALTER TEXT SEARCH CONFIGURATION public.english_nostop - ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, hword, hword_part, word WITH english_stem_nostop; -``` - -So in query to find text, it have some little change: - -```sql -SELECT * FROM tweets WHERE vector @@ to_tsquery('english_nostop',REPLACE(LOWER('multiple words with no order'),' ', ' & ')); -``` - -## Result - -It will be no sense if the result (performance) of this approach isn't better than the normal way which uses `LIKE` operator. So, let have a comparison: - -### LIKE operator - -```sql -EXPLAIN ANALYZE SELECT * FROM tweets WHERE content ILIKE '%needles%' OR content ILIKE '%well%'; -``` - -| QUERY PLAN | -| --------------------------------------------------------------------------------------------------------------- | -| Seq Scan on tweets (cost=0.00..134808.31 rows=10232 width=1257) (actual time=1.744..1384.645 rows=6502 loops=1) | -| Filter: (((content)::text ~~_ '%needles%'::text) OR ((content)::text ~~_ '%well%'::text)) | -| Rows Removed by Filter: 200497 | -| Planning Time: 1.777 ms | -| Execution Time: 1445.618 ms | - -### Full-text search - -```sql - -EXPLAIN ANALYZE SELECT * FROM tweets WHERE vector @@ to_tsquery(REPLACE(LOWER('needles well'),' ', ' & ')); -``` - -| QUERY PLAN | -| --------------------------------------------------------------------------------------------------------------------- | -| Bitmap Heap Scan on tweets (cost=44.42..137.34 rows=22 width=1257) (actual time=0.094..0.135 rows=4 loops=1) | -| Recheck Cond: (vector @@ to_tsquery('needles & well'::text)) | -| Heap Blocks: exact=4 | -| -> Bitmap Index Scan on idx_tweet_vector (cost=0.00..44.41 rows=22 width=0) (actual time=0.074..0.081 rows=4 loops=1) | -| Index Cond: (vector @@ to_tsquery('needles & well'::text)) | -| Planning Time: 0.230 ms | -| Execution Time: 0.231 ms | - -The result shows that the planning time and execution time of `LIKE` operator are worse than `FTS` method. Because the demo is just an illustration of 200k records table. For a larger table (millions of records), the performance of using `LIKE` operator is much worse than using `FTS` method. Furthermore, you can notice that, with `FTS` we can search words with no orders required when in `LIKE` operator method, the words' order is also counted to the result. - -## Note - -Examples in this post are demonstrated with `Postgresql >= 11.x` and use `tweets` table below with over 200k rows. - -```sql -| tweets | -|------------------| -| id(int) | -| content(VARCHAR) | -| vector(tsvector) | -``` - -## References - -- https://pganalyze.com/blog/gin-index -- https://wiki.postgresql.org/images/2/25/Full-text_search_in_PostgreSQL_in_milliseconds-extended-version.pdf -- https://www.compose.com/articles/mastering-postgresql-tools-full-text-search-and-phrase-search/ -- SQL Performance Explained by Markus Winand (Book) diff --git a/topics/data/google-data-fusion.md b/topics/data/google-data-fusion.md deleted file mode 100644 index 4dde15b44..000000000 --- a/topics/data/google-data-fusion.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -title: Google data fusion -description: Learn how to use Google Cloud Data Fusion to build and manage ETL pipelines with a simple drag-and-drop interface, integrating data from cloud storage to BigQuery for fast analytics. -date: 2024-01-22 -github_id: mickwan1234 -icy: 10 -redirect: - - /qzdulw ---- - -## Introduction - -Google Data Fusion is a fully managed, cloud-native data integration service that enables users to efficiently build and manage ETL/ELT data pipelines. It is designed to streamline data engineering tasks for enterprise users and is built on top of the open-source project CDAP - -. Key features and benefits of Google Data Fusion include: - -- **Graphical user interface**: Data Fusion offers a convenient drag-and-drop interface for building data pipelines, making it easier for users to focus on insights and actions rather than writing code. -- **Flexibility**: While the UI provides a quick and easy way to create pipelines, Data Fusion is also extensible, allowing users to add custom code when needed**[2](https://gcloud.devoteam.com/blog/what-is-google-cloud-data-fusion/)**. -- **GCP-native architecture**: Data Fusion is fully integrated and supported by Google Cloud Platform, unlocking the scalability, reliability, security, and privacy guarantees of Google Cloud. -- **Data governance**: Data Fusion includes built-in features for data governance, such as end-to-end data lineage, integration metadata, and cloud-native security and data protection services. -- **Integration with Google Cloud services**: Data Fusion connects with various Google Cloud services, such as Cloud Storage, BigQuery, and Cloud Spanner, allowing users to create, manage, and analyze data in a centralized manner. - -Overall, Google Data Fusion simplifies data engineering tasks and enables users to focus on data analytics and deriving insights for better customer service and operational efficiency - -## Use cases - -Google Cloud Data Fusion is a powerful tool for building and managing data pipelines, and its use cases are diverse and expanding. Here are some of the most common and impactful ways organizations are using it: - -**Data Integration and ETL:** - -- **Consolidating data from multiple sources:** Data Fusion simplifies data ingestion from various on-premises, cloud, and SaaS applications, unifying it into a single repository for analysis. Examples include CRM, ERP, social media, and sensor data. -- **Transforming and enriching data:** Data Fusion offers a wide range of data transformation tools, including filtering, sorting, aggregation, and joining, enabling you to clean, enrich, and prepare data for specific use cases. -- **Building data warehouses and lakes:** Data Fusion seamlessly integrates with Google BigQuery and other data storage solutions, facilitating the creation of centralized data repositories for analytics and reporting. - -**Data Migration and Modernization:** - -- **Migrating data to the cloud:** Data Fusion simplifies the process of migrating on-premises data to Google Cloud, providing a managed and scalable solution for organizations looking to modernize their data infrastructure. -- **Replicating data between cloud platforms:** Data Fusion enables seamless data replication between different cloud environments, including Google Cloud Platform, AWS, and Azure, ensuring data consistency and availability. - -**Real-time Data Processing and Analytics:** - -- **Building real-time data pipelines:** Data Fusion supports Apache Spark, allowing you to create real-time data pipelines for near-instantaneous insights and decision-making based on streaming data feeds. -- **Enabling real-time dashboards and reports:** Data Fusion can be used to feed data into real-time dashboards and reports, providing stakeholders with immediate visibility into critical business metrics. -- **Triggering actions based on data events:** Data Fusion can be used to trigger actions based on specific data events, such as sending alerts, updating records, or initiating workflows in other systems. - -**Additional Use cases:** - -- **Data governance and compliance:** Data Fusion offers features like data lineage tracking and auditing, ensuring data quality and compliance with regulations. -- **Self-service data access:** Data Fusion empowers business users with self-service data access capabilities, allowing them to explore and analyze data without needing technical expertise. -- **Building custom data connectors:** Data Fusion provides a framework for building custom connectors to integrate with niche applications and data sources. - -These are just a few examples, and the possibilities for using Data Fusion are vast. Its flexibility, scalability, and ease of use make it a valuable tool for organizations looking to unlock the potential of their data. - -## Pricing - -Google Data Fusion have some pricing tier in the following table: -![](assets/google-data-fusion-20240122120201050.webp) - -Comparison between the tiers: -![](assets/google-data-fusion-20240122120216075.webp) - -## How to setup Data fusion: - -### Prerequisites: - -1. **Google Cloud Platform account:** Ensure that you have a Google Cloud Platform (GCP) account. If you don't have one, you can sign up [here](https://cloud.google.com/). -2. **Enable the Cloud Data Fusion API:** In the GCP Console, navigate to the API & Services > Dashboard. Search for "Cloud Data Fusion API" and enable it. -3. **Install and configure Google Cloud SDK:** Install the [Google Cloud SDK](https://cloud.google.com/sdk) on your local machine. After installation, run **`gcloud init`** to set up your credentials and project. - -### Steps to Set Up Google Cloud Data Fusion: - -1. **Create a Cloud Storage bucket:** Create a Cloud Storage bucket to store the artifacts and metadata required by Cloud Data Fusion. Replace **`[BUCKET_NAME]`** with your desired bucket name. - - ```bash - bashCopy code - gsutil mb -l [REGION] gs://[BUCKET_NAME] - - ``` - -2. **Create a Cloud Data Fusion instance:** Use the following command to create a Cloud Data Fusion instance. Replace **`[INSTANCE_NAME]`** with your desired instance name, and **`[REGION]`** with the region where you want to deploy the instance. - - ```bash - bashCopy code - gcloud data-fusion instances create [INSTANCE_NAME] \\ - --region=[REGION] \\ - --zone=[ZONE] \\ - --network=[NETWORK_NAME] \\ - --subnet=[SUBNET_NAME] \\ - --bucket-uri=gs://[BUCKET_NAME]/[DIRECTORY] - - ``` - - - **`-zone`**: Specify the zone for the instance. - - **`-network`**: Specify the VPC network name. - - **`-subnet`**: Specify the subnet within the network. - - **`-bucket-uri`**: Specify the Cloud Storage bucket URI. - -3. **Access Cloud Data Fusion UI:** After the instance is created, you can access the Cloud Data Fusion UI using the generated endpoint. Navigate to the URL displayed in the command output. - - ```bash - bashCopy code - gcloud data-fusion instances describe [INSTANCE_NAME] --region=[REGION] - - ``` - -4. **Connect to the Cloud Data Fusion UI:** Open the provided URL in a web browser to access the Cloud Data Fusion UI. Log in using your Google Cloud credentials. -5. **Explore and create pipelines:** Once you're in the Cloud Data Fusion UI, you can start exploring and creating ETL pipelines using the visual interface. - -Remember to replace placeholders like **`[REGION]`**, **`[BUCKET_NAME]`**, **`[INSTANCE_NAME]`** - -## How to setup a simple ETL with Google Data Fusion ( with GCP console): - -In this demo we’ll setup a simple ETL pipeline to import, transform load a csv file to a Google Big query data table - -### Step 1: go to the Big Query console page and create a dataset and table: - -- On the top left corner of the main page of Big query select add and add a dataset. Then click on the three dot simple on the left side of the data set name to add a table. Right after that. you should see this screem to add a table. - - ![](assets/google-data-fusion-20240122120312319.webp) -- Fill in the table name (the only mandatory field that left blank after you do the previous step). You can tweak others setting if you want to. Then click Create Table. - -## Step 2: go to your Google Data Fusion instance - -- Go to your Data Fusion page and locate your data fusion instance. You should be see it listed right after you go the main page of Data fusion console. -- Click on the instance you should be able to see a prompt like this: - - ![](assets/google-data-fusion-20240122120335588.webp) -- Click the View Instance link. You should see the landing page of Google Data Fusion instance. Click on Studio button to start designing pipelines. You should see a screen like this: - - ![](assets/google-data-fusion-20240122120406512.webp) - -## Step 3: preparing you CSV file on GCS - -- Create a bucket and push you CSV to the storage. I prepared mine like this. - - ![](assets/google-data-fusion-20240122120426619.webp) - -### Step 4: start designing your first pipeline - -- We’ll create a GCS as a source of data so on the search bar, search for ‘gcs’ and drag the ‘GCS’ simple in the `Source` section to the canvas - - ![](assets/google-data-fusion-20240122120526652.webp) -- Click on `Properties` and you should be able to see a screen like this. - - ![](assets/google-data-fusion-20240122120552908.webp) -- Click on `Browse` to browse through the files you uploaded to GCS. Locate your csv data source file and select it. - - ![](assets/google-data-fusion-20240122120615991.webp) -- (Optional) Enable the `Enable Quoted Values` and `Use First Row as Header` toggles and click on `Get Schema`. You should able to find your data schema on the right side of the screen - - ![](assets/google-data-fusion-20240122120647202.webp) -- You finished setting up the source CSV. Go on and click the `X` button on the top right next to `Validate` button. -- Next, you’ll need to transform data to match you expected schema. Search for `wrangler` on the search bar and drag out `Wrangler` in the `Transform` section and drag an arrow from `GCS` box to it for Wrangler to recognize the input schema. - - ![](assets/google-data-fusion-20240122120708508.webp) -- Then click on `properties` of the the `Wrangler` box. - - ![](assets/google-data-fusion-20240122120734801.webp) -- Here you can transform your data. Defines you output schema on the left and properties in the middle section. Here I’ll just left it like it is since it a simple demo. -- Click on the top right `X` button to go back to the canvas. -- Next, you’ll need a storage for you output data. Search `bigquery` in the search box and drag out `Bigquery` box from the `Sink` section and connect it to the `Wrangler` box. - - ![](assets/google-data-fusion-20240122120800866.webp) -- Click on `Properties` of the Bigquery box you’ll find this screen. - - ![](assets/google-data-fusion-20240122120820157.webp) -- You can find you input data schema on the left side of the screen. Here click on `Browse` to locate your Bigquery data table that you created in step 1 (Mine is `annual_enterprise_survey` in this case) and select it by double click. - - ![](assets/google-data-fusion-20240122120841121.webp) -- (Optional) you can force the destination table of match you input schema by turn on these toggle. - - ![](assets/google-data-fusion-20240122120859634.webp) -- Click on the `X` again to go back to the canvas. You completed your first pipeline. - -### Step 5: deploy and run your pipeline - -- Naming your pipeline ( the name must be unique in the instance). Click on the `Name your pipeline` text on the top left of the screen. - - ![](assets/google-data-fusion-20240122120943004.webp) -- Then click on `Deploy` button on the top right to deploy the pipeline. - - ![](assets/google-data-fusion-20240122120958563.webp) -- Wait for the deploy process to complete. You will face this screen - - ![](assets/google-data-fusion-20240122121014344.webp) -- Click on the `Run` button at the top center of the screen and wait for it to execute pipelines. - - ![](assets/google-data-fusion-20240122121132342.webp) -- After the the status show `Succeeded` you can check your destination table for the output data. - - ![](assets/google-data-fusion-20240122121149587.webp) -- Here is the end result data in your Bigquery data table. - - ![](assets/google-data-fusion-20240122121206926.webp) - -And that’s it, you’re done. Congrat on your first data pipeline with Google Data Fusion. diff --git a/topics/data/google-dataproc.md b/topics/data/google-dataproc.md deleted file mode 100644 index b5409ff3b..000000000 --- a/topics/data/google-dataproc.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -title: Google dataproc -description: Learn how to set up and use Google Dataproc for fast, scalable Apache Spark and Hadoop data processing on Google Cloud, including cluster creation, job submission, and pricing details. -date: 2024-01-22 -github_id: mickwan1234 -icy: 10 -redirect: - - /LrNxPQ ---- - -## Introduction - -Google Dataproc is a fast and easy-to-use, fully managed cloud service for running Apache Spark and Hadoop workloads. It allows users to process and analyze large-scale data efficiently and at scale. Key features and benefits of Google Dataproc include: - -- **Fast and scalable processing**: Dataproc enables users to run Apache Spark and Hadoop workloads on Google Cloud Platform, taking advantage of its scalable infrastructure and resources. -- **Integration with Google Cloud services**: Dataproc integrates seamlessly with other Google Cloud services, such as BigQuery, Cloud Storage, and Cloud Spanner, allowing users to access and analyze data in a centralized manner**[1](https://cloud.google.com/dataproc)** -- **Data analytics and machine learning**: Dataproc supports data scientists and engineers in building and deploying machine learning models, as well as analyzing data using custom-built applications and open-source libraries. -- **Managed and secure environment**: Google Dataproc provides a fully managed and secure environment for running data processing workloads, ensuring that users can focus on their data analysis and insights without worrying about infrastructure management**[1](https://cloud.google.com/dataproc)**. - -Overall, Google Dataproc simplifies big data processing and analysis by providing a fast, scalable, and integrated platform for running Apache Spark and Hadoop workloads on Google Cloud Platform. - -## Use cases - -Google Cloud Dataproc has a wide range of use cases for processing large datasets and running big data applications. Here are some of the most popular examples: - -**Batch processing:** - -- **Data warehousing and ETL:** Dataproc can be used to extract, transform, and load (ETL) data from various sources into data warehouses like BigQuery. It can also be used to process and clean large datasets for data analysis. -- **Log processing:** Dataproc can be used to analyze massive log files from web applications, servers, and other systems to identify trends and troubleshoot issues. -- **Financial analysis:** Dataproc can be used to analyze large financial datasets for fraud detection, risk assessment, and portfolio management. -- **Scientific simulations:** Dataproc can be used to run large-scale scientific simulations in fields like physics, chemistry, and biology. -- **Image processing:** Dataproc can be used to process and analyze large collections of images and videos for applications like object recognition, facial recognition, and image classification. - -**Real-time processing:** - -- **Data streaming:** Dataproc can be used to process data streams in real-time for applications like fraud detection, anomaly detection, and event processing. -- **Internet of Things (IoT):** Dataproc can be used to analyze data from IoT devices in real-time for applications like environmental monitoring, predictive maintenance, and smart city management. -- **Machine learning:** Dataproc can be used to train and deploy machine learning models on large datasets. - -**Other use cases:** - -- **Web scraping:** Dataproc can be used to scrape data from websites at scale. -- **Social media analysis:** Dataproc can be used to analyze large social media datasets for sentiment analysis, topic modeling, and social network analysis. -- **Genomics:** Dataproc can be used to analyze large genomic datasets for research and clinical applications. - -## Pricing - -[Dataproc on Compute Engine](https://cloud.google.com/dataproc) pricing is based on the size of Dataproc clusters and the duration of time that they run. The size of a cluster is based on the aggregate number of [virtual CPUs (vCPUs)](https://cloud.google.com/compute/docs/machine-types) across the entire cluster, including the master and worker nodes. The duration of a cluster is the length of time between cluster creation and cluster stopping or deletion. - -The Dataproc pricing formula is: `$0.010 * # of vCPUs * hourly duration`. - -Although the pricing formula is expressed as an hourly rate, Dataproc is billed by the second, and all Dataproc clusters are billed in one-second clock-time increments, subject to a 1-minute minimum billing. Usage is stated in fractional hours (for example, 30 minutes is expressed as 0.5 hours) in order to apply hourly pricing to second-by-second use. - -Dataproc pricing is in addition to the [Compute Engine per-instance price](https://cloud.google.com/compute/pricing) for each virtual machine (see [Use of other Google Cloud resources](https://cloud.google.com/dataproc/pricing#use_of_other_google_cloud_resources)). - -As an example, consider a cluster (with master and worker nodes) that has the following configuration: - -| Item | Machine Type | Virtual CPUs | Attached persistent disk | Number in cluster | -| ------------ | ------------- | ------------ | ------------------------ | ----------------- | -| Master Node | n1-standard-4 | 4 | 500 GB | 1 | -| Worker Nodes | n1-standard-4 | 4 | 500 GB | 5 | - -This Dataproc cluster has 24 virtual CPUs, 4 for the master and 20 spread across the workers. For Dataproc billing purposes, the pricing for this cluster would be based on those 24 virtual CPUs and the length of time the cluster ran (assuming no nodes are scaled down or preempted). If the cluster runs for 2 hours, the Dataproc pricing would use the following formula: - -`Dataproc charge = # of vCPUs * hours * Dataproc price = 24 * 2 * $0.01 = $0.48` - -In this example, the cluster would also incur charges for Compute Engine and Standard Persistent Disk Provisioned Space in addition to the Dataproc charge (see [Use of other Google Cloud resources](https://cloud.google.com/dataproc/pricing#use_of_other_google_cloud_resources)). The [billing calculator](https://cloud.google.com/products/calculator) can be used to determine separate Google Cloud resource costs. - -## How to setup a Dataproc cluster - -### Prerequisites: - -1. **Google Cloud Platform account:** - - You need a GCP account. If you don't have one, you can sign up for a free trial at [Google Cloud Console](https://console.cloud.google.com/). -2. **Enable billing:** - - Make sure you have billing enabled for your GCP project. -3. **Install Google Cloud SDK:** - - Install the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install) on your local machine. This SDK includes the **`gcloud`** command-line tool, which you'll use to interact with GCP. -4. **Create a GCP project:** - - Create a GCP project if you don't already have one. You can do this through the [GCP Console](https://console.cloud.google.com/). -5. **Enable APIs:** - - Enable the necessary APIs for Dataproc and other services. You can do this through the [APIs & Services > Dashboard](https://console.cloud.google.com/apis/dashboard) section in the GCP Console. - -### Set Up a Dataproc Cluster: - -Now that you have the prerequisites in place, you can proceed to set up a Dataproc cluster: - -1. **Open Cloud Shell:** - - - Open the [Google Cloud Console](https://console.cloud.google.com/). - - Click on the "Activate Cloud Shell" button in the upper right corner. - -2. **Create a Dataproc cluster:** - - - Use the following **`gcloud`** command to create a Dataproc cluster. Adjust the parameters as needed. - ```bash - bashCopy code - gcloud dataproc clusters create CLUSTER_NAME \\ - --region REGION \\ - --num-workers NUM_WORKERS \\ - --worker-machine-type MACHINE_TYPE \\ - --master-machine-type MASTER_MACHINE_TYPE - ``` - Replace the placeholders (**`CLUSTER_NAME`**, **`REGION`**, **`NUM_WORKERS`**, **`MACHINE_TYPE`**, **`MASTER_MACHINE_TYPE`**) with your desired values. - -3. **Access the cluster:** - - - Once the cluster is created, you can access the master node using SSH. You can find the SSH command in the Dataproc cluster details in the GCP Console. - -4. **Run jobs:** - - - Submit jobs to your Dataproc cluster using tools like Apache Spark or Hadoop. You can submit jobs using the **`gcloud dataproc jobs submit`** command. - -5. **Delete the cluster:** - - - When you're done with the cluster, remember to delete it to avoid incurring unnecessary costs. - ```bash - bashCopy code - gcloud dataproc clusters delete CLUSTER_NAME --region REGION - ``` - Replace the placeholders with your cluster's name and region. - -## How to deploy a Spark job on Dataproc (Using Google cloud console) - -### Step 1: In the created cluster click on `Submit Job` button on the top right of the main console screen - -![](assets/google-dataproc-20240122130334062.webp) - -### Step 2: Fill all necessary field in the job creation modal: - -![](assets/google-dataproc-20240122130408379.webp) - -- Job ID: job unique name across the cluster -- Job Type: the type of the job that you submit to Dataproc including (in this demo we’ll use PySpark): - ![](assets/google-dataproc-20240122130427570.webp) -- Main file: the directory of main job file from GCS or HDFS or inside the compute engine. -- All others field is optional. -- After you fill in the necessary input i should be something like this: - ![](assets/google-dataproc-20240122130457028.webp) - -- Then press `Submit` to submit the job - -### Step 3: - -After submitting the job you should be able to similar screen to this: -![](assets/google-dataproc-20240122130525347.webp) - -Wait for the job status change to succeeded (or failed): -![](assets/google-dataproc-20240122130542001.webp) - -And that is how you run a Spark ( or Hadoop) job on Google Cloud Dataproc. -Full video step by step: -![[_assets/Google Dataproc/830c086903ad2dcaf7bdece0af8e3647_MD5.mp4]] diff --git a/topics/data/hadoop-distributed-file-system-hdfs.md b/topics/data/hadoop-distributed-file-system-hdfs.md deleted file mode 100644 index 0a4ac3b58..000000000 --- a/topics/data/hadoop-distributed-file-system-hdfs.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -title: Hadoop distributed file system hdfs -description: Learn why Hadoop Distributed File system HDFS is essential for big data, enabling block-based storage, fault tolerance through replication, and scalable distributed file management across clusters. -date: 2022-06-15 -github_id: dudaka -redirect: - - /unxtGQ ---- - -## HDFS - Why Another FileSystem? - -HDFS (Hadoop Distributed File system) takes care of all the storage related complexities in Hadoop. Why is there a need for another file system like HDFS? - -### File system - -![](assets/hadoop-distributed-file-system-hdfs_file-system-hdfs.png) - -File system is an integral part of every operating system, it basically governs the storage in your hard disk. For example, you give a person a book and you give another person pile of unordered papers from the same book, then ask each of them to go to chapter 34. Who do you think will get to chapter 34 faster? The one with the book because he can simply go to the index, look for chapter 34 look up the page number and go to the page. Whereas the one with the pile of papers has to go through the pile of papers and if he is lucky he might find chapter 34. Just like a well-organized book, a file system helps to navigate the data that is stored in your storage. Without the file system, the information stored in your hard disk will be one large body of data, but no way to tell where one piece of information stops and the next begins. - -There are some of the major functions of a file system. File system controls how the data is stored and retrieved. Basically, when you read and write files to your hard disk your request goes through a file system. Next, file system has the metadata about your files and folders. Metadata information like file name, size, owner, created/modified time, etc. File system also takes care of permissions and security. File system manages your storage space, so when you ask to write a file to hard disk file system helps figure out where in the hard disk it should write the file. And it should write the file as efficiently as possible. - -### Different file systems - -The most legendary file system from Microsoft is FAT32. Maximum file size that a Fat32 file system can support is 4GB. If we have a file which is 5GB in size, we're out of luck with FAT32 and it has a 32GB volume limit or a logical drive limit. Thefore, our drive can be of size 32GB and not more with FAT32. The size limits can be more or less based on the file system configuration. So, if we use windows 95 or 98, we probably use FAT32. - -Next generation file system from Windows after FAT32 is NTFS (New Technology File system) and it supports 16 Exabyte file and volume limit of 16 Exabyte, that is a very huge number, which is 1024 Petabytes. Therefore, NTFS can clearly support huge volume of data. Starting Windows Server 2012, Windows introduced ReFS (Resilient File system). - -How about file systems from MAC? HFS (Hierarchical File system) is a legacy file system from Mac Apple that started using HFS+ from MAC OS 8.1 and above. For example, if we used iPod, we would have used HFS+. HFS+ can also handle a huge volume of data up to 8 Exabytes in size. - -Now, that is about Linux, ext is the most popular file system in Linux. ext3 is the third generation file system in use since 2001, then came ext4. ext4 can support individual file sizes up to 16 Terabytes and volumes up to 1 Exabyte. Next, XFS is created by Silicon Graphics and it can support up to 8 exabytes in file and volume limit. We look up your file system in Linux with command `df -T`. - -Clearly, we have file systems where we can store big data sets. Then, what is the need for HDFS? From section Understanding Big Data Problem of [Apache Hadoop and Big Data article](apache-hadoop-and-big-data.md), we know that to support truly parallel computation, we had to divide the data set into blocks and store them in different nodes And to recover from data loss, we also replicated each block in more than one node. - -![](assets/hadoop-distributed-file-system-hdfs_hadoop-distributed-file-system.png) - -Assume that we have a 10 node cluster and we have ext4 as the file system on each node like the above image. We will refer ext4 on each node as the local file system. The first task of our proposed file system is when we upload a file to this proposed file system, we need the file system to divide the data set into fixed size, i.e. blocks. Although every file system has a concept of blocks, the concept of blocks and HDFS is very different when compared to the blocks and traditional file systems. We will see the differences in other article :)). - -Next, our file system should have a distributed view of the files or blocks in the cluster which is not possible with our local file system which is ext4. As shown in the above image, our local ext4 file system on Node 1 has no idea what is on Node 2. Similarly, Node 2 has no idea of what is in Node 1. Since the ext4 file systems in both Node 1 and Node 2 are local to each node, there is no way they can have a global or distributed view of the entire 10 node cluster. That is why we say the ext4 on individual nodes as local file systems. Next important thing is replication which adds a lot of complexity. Since ext4 in Node 1 has no idea about storage in any other node, it does not have the ability to replicate blocks in Node 1 to the other nodes. Therefore, we are exposed to data loss. - -Now, assume we have a file system on top of ext4 but only this time it spreads across all the nodes. We call that file system, Hadoop Distributed File system Then, when you upload a file to HDFS it will automatically be split into 128MB-fixed size blocks. In the older versions of Hadoop, the file was divided into 64MB-fixed size blocks. HDFS takes care of placing the blocks in different nodes and also take care of replicating each block into more than one node. By default, hdfs replicates a block to three nodes. If we copy a 700 MB dataset into HDFS, HDFS will divide the data set into 128MB blocks. Thus, we will have 5 equal sized 128MB block and one 60MB block. - -Since hdfs has a distributed view of the cluster, it can easily decide which nodes should hold these 6 blocks and also pick the nodes to hold the replicated blocks HDFS will continue to creep track of all the blocks and their node assignments all the time. So when a user asked about the 700 MB data set, it knows how to construct the file from the blocks. - -HDFS, by no means, is a replacement for the local file system. Our operating systems still rely on the local file system. In fact, the operating system does not care about the presence of HDFS. One more interesting thing, HDFS should still go through ext4 to save the blocks in the storage. Hence, HDFS is placed on top of the local file system. - -The true power of HDFS is that it is spread across all the nodes in the cluster and it has a distributed view of the cluster. And hence it knows how to construct the 700 MB data set in the example from the underlying blocks whereas the ext4 does not have a distributed view and only knows about the blocks in its storage that it is managing. - -### Benefits of HDFS - -- HDFS supports the concept of blocks: When you upload a file into HDFS, the file is divided into fixed size blocks to support distributed computation and that is the key for Hadoop. Also HDFS keeps track of all the blocks in the cluster -- Data failures or data corruption are inevitable in any big data environment, even in small environments. HDFS maintains data integrity and help recover from data loss by replicating the blocks in more than one node. -- HDFS supports scaling: if we like to expand our cluster by adding more nodes, it's very easy to do with HDFS. -- Cost effective: we don't need any specialized hardware to run or operate HDFS and this is very important because we are refering about potentially hundreds of nodes. HDFS was built ground up to work with commodity computers. - -## Blocks - -All files will be divided into blocks and will be replicated three times by default across the nodes in the cluster in HDFS. Let's do an experiment on Windows, we can create a very small text file, name it as `test.txt`, then add just some senctences into the text file and save the text file. Now, when we right-click and click the properties of the text file. A popup will appear as shown as the below photo. - -![](assets/hadoop-distributed-file-system-hdfs_file-size-vs-block-size.png) - -The size of the file is 1.34 KB but the size on the disk for this file is 4 KB. Why the file is taking 4 KB on the disk when the actual size of the file is only 4 bytes? Because 4 KB is the block size or cluster size of the operating system in our computer which is NTFS. So 4 KB are multiples of 4 KB is the minimum amount of space that the file system will assign to a file. If the file size is 2 KB it will still take up 4 KB on disk leaving the 2kb unused and this 2 KB cannot be reused for anything else, leaving them unused forever. If the file size is 8 KB it will take up 8 KB that is 2 4KB-blocks. And if the file size is 13 KB, it will take up 16 KB or 4 blocks leaving 3 KB unused again this 3 KB cannot be reused, leaving them unused forever. Therefore, 4 KB are multiples of 4 KB is the minimum amount of space the file system can assign at any given. - -Now considering to HDFS, the configured block size of your hdfs in your Hadoop cluster is 256 MB. And we uploaded a file which is 1 MB in size. We may guess that HDFS allocates 256 MB to store a 1MB file leaving 255 MB unused. That is not correct. It would be a lot of space wasted. - -As mentioned from the previous section, we know that HDFS is not a replacement to the local file system and all the blocks are physically stored in the local file system even though the file uploaded to HDFS is divided into blocks. These blocks are stored in the hard disk which is managed by the local file system, which means in the hard disk, the file will be stored as per the block size of the local file system. So if we have a cluster in which the local file system is ext4 for instance and with a block size of 4 KB to store a 1MB file which is 1024 KB, we would need 256 of 4KB blocks which is exactly 1 MB to store the file in ext4. If the file size is 1025 KB for instance, we would need 257 of 4 KB blocks leaving 3 KB in the last block unused. Even though HDFS has a block size, the space allocated for the file on the disk will still obey the rules of the local file system. For that reason, HDFS will not allocate 256 MB to store a 1MB file. - -There are some questions. If the blocks are stored by underlying local file system. Why are HDFS block size so huge like 128 MB or 256 MB as compared to 4 KB of block size in the local file system? Why don't we keep it as 4 KB? When the block size is huge, the OS will attempt to store the file in contiguous blocks on the disk. If the blocks are in contiguous locations, both reads and writes will be faster because the blocks are laid out next to each other as the disk head doesn't have to seek and position itself over and over again for blocks as they are stored contiguously. This is a huge benefit as the read and writes will be very efficient. However, it is very important to note that OS will attempt to write big files in contiguous location but it does not guarantee that. - -If storing big files are advantageous in terms of read and write efficiency, why do we have to split the files into blocks at all? Why don't we store the file as a whole? By dividing the files into blocks, we can store data set of any size and not limited to size of the volume of any individual hard disk. As mentioned in the previous section, each file system has its own volume limit. For example, NTFS file system has a volume limitation of 16 Exabytes. So if we have a data set which is 17 Exabytes in size. We cannot store that file on a hard disk as a whole. But by dividing the data set into blocks, we can store the blocks across many nodes in the cluster. Also dividing a data set to blocks and replicating the blocks offer redundancy and fault tolerance. - -Asuming that we have a file which is well within the size of the hard disk and we decide to store the file as a whole instead of dividing them into blocks and decide to replicate the file as a whole three times. The below image illustrate that. Now we have one file replicated into three nodes: Node 1, Node 2 and Node 3. In an event where all three nodes crash, we would lose the file. - -![assets/hadoop-distributed-file-system-hdfs_3-crashes.png](assets/hadoop-distributed-file-system-hdfs_3-crashes.png) - -We can consider to another scenario where we divide the same file and into five blocks and store each block in a different node with the replication factor of 3. The blocks could be stored as much as up to 15 nodes across the cluster, as shown in the below image. Hence, to physically lose the file all 15 nodes would have to crash, which is less likely as compared to three nodes going down at once. - -![](assets/hadoop-distributed-file-system-hdfs_15-crashes.png) - -## Working with HDFS - -There are some well-known commands to work with a local file system in Linux. - -``` -ls - to list a content in a directory -mkdir - to create a directory -cp - to copy -mv - to move -rm - to delete -``` - -And there are some HDFS commands which always start with `hdfs`. - -Listing root directory - -``` -hadoop fs -ls / -``` - -Listing default to home directory - -``` -hadoop fs -ls -hadoop fs -ls /user/dungho -``` - -Create a directory in HDFS - -``` -hadoop fs -mkdir hadoop-test1 -``` - -Copy from local FS to HDFS - -``` -hadoop fs -copyFromLocal /tmp/stocks.csv hadoop-test1 -``` - -Copy from HDFS to local FS - -``` -hadoop fs -copyToLocal hadoop-test1/stocks.csv . - -hadoop fs -ls hadoop-test1 -``` - -Create 2 more directories - -``` -hadoop fs -mkdir hadoop-test2 - -hadoop fs -mkdir hadoop-test3 -``` - -Copy a file from one folder to another - -``` -hadoop fs -cp hadoop-test1/stocks.csv hadoop-test2 -``` - -Move a file from one folder to another - -``` -hadoop fs -mv hadoop-test1/stocks.csv hadoop-test3 -``` - -Check replication - -``` -hadoop fs -ls hadoop-test3 -``` - -Change or set replication factor - -``` -hadoop fs -Ddfs.replication=2 -cp hadoop-test2/stocks.csv hadoop-test2/test_with_rep2.csv - -hadoop fs -ls hadoop-test2 - -hadoop fs -ls hadoop-test2/test_with_rep2.csv -``` - -Changing permissions - -``` -hadoop fs -chmod 777 hadoop-test2/test_with_rep2.csv -``` - -File system check - requires ad previleges - -``` -sudo -u hdfs hdfs fsck /user/dungho/hadoop-test2 -files -blocks -locations - -sudo -u hdfs hdfs fsck /user/dungho/hadoop-test3 -files -blocks -locations - -sudo -u hdfs hdfs fsck /user/ubuntu/input/yelp/yelp_academic_dataset_review.json -files -blocks -locations - -vi /etc/hadoop/conf/hdfs-site.xml - -/data/1/dfs/dn/current/BP-2125152513-172.31.45.216-1410037307133/current/finalized - -``` - -Delete directories/files in HDFS - -``` -hadoop fs -rm hadoop-test2/test_with_rep5.csv - -hadoop fs -rm -r hadoop-test1 -hadoop fs -rm -r hadoop-test2 -hadoop fs -rm -r hadoop-test3 -``` - -## HDFS - read & write - -Copying from local to HDFS does a write operation to HDFS because from the local file system we are writing a file into HDFS. Whereas copy from HDFS to local does a read operation because it reads a file from HDFS and write it to the local file system. We know that a file or data set is divided into chunks of blocks and stored across the nodes in the cluster. Imagining that we are the client and we are trying to read a file from HDFS and how do we know where the blocks are physically stored? - -![](assets/hadoop-distributed-file-system-hdfs_hadoop-nodes.png) - -A Hadoop cluster has two types of nodes. The first type of node and the most important node in the hadoop cluster is called the name node, also known as the master. The second type of node is known as the data node also known as the slave. - -The master node has the metadata of HDFS meaning it has all the infomation about: list of files, the list of blocks, who created the files, when a file got created, when it was modified, the permission of the files, etc. In other words, it has all the information about hdfs and what is in HDFS. Hence, name node is a very important node in the cluster. Name node does not store the actual files or data sets. The files or data sets are stored in another type of nodes called the data nodes data nodes also known as the slaves stores the physical blocks for the files in HDFS. Usually there is only one active name node in the cluster and we can have as many data nodes as we like in the cluster depending on the amount of data we would like to store in HDFS. In short, if we take a file in HDFS, the data nodes will store the actual physical blocks for that file. Whereas the name node knows the list of blocks that make up the file and the list of data nodes that stores the blocks for that file and also information about the file like who created it, when it was created, its permissions, etc. - -### Read operation - -For example, we want to read a file from HDFS and the file is made up of 10 blocks. We execute `hadoop fs -copyToLocal` command from one of the data nodes in the cluster to copy the file from HDFS to the local file system. When we execute `hadoop fs -copyToLocal`, a Java program is executed behind the scenes which does a series of operations to read the file from HDFS. All these operations happens behind the scenes and it is intransparent to us. - -![](assets/hadoop-distributed-file-system-hdfs_copy-operation.png) - -The above image shows step by step what happens behind the scenes. First the client program trying to read the file will contact the name node to get the list of block locations for the file. If the replication factor for the file is 3, then for each block, the name node will return address of all three data nodes that stores a copy of the actual block. When the name node returned the list of data nodes that has a copy for each block, it also sorts the data node in terms of proximity to the client requesting the read. Thus, the client can read the block from the closest data node. - -#### What is node proximity? - -If you have 100 nodes in the cluster, how does the client know which node is the closest one in a Hadoop cluster? The data nodes are physically organized into racks, the data nodes in a rack are connected to one another and all the racks in the cluster are connected to one another. - -![](assets/hadoop-distributed-file-system-hdfs_node-proximity.png) - -Let's consider to the example showing in the above image, we have two racks: Rack 1 and Rack 2. And our client who is trying to read the file is running on Rack 1 - Data Node 2. There will be 3 replicas for a block: for example, the first replica is stored in Rack 1 - Data node 2, the second replica is stored in Rack 1 - Data node 6 and the third replica in Rack 2 - Data node 5. Now Rack 1 - Data Node 2 is considered the closest data node to the client because the client is also running on the same node. The next closest node to the client will be the node which is on the same rack, in this case, it will be Rack 1 - Data Node 6. The next closest node to the client will be the node which is on a different rack in this case it will be Rack 2 - Data node 5. If the client runs on a data node which does not hold a copy of the block and if we have two data nodes on the same rack which holds the replica of the block then one of the data nodes from the same rack will be chosen at random. In essence, the closest node to the client is a node on the same rack as the client and the next closest node is a node on a different rack. For the read operation, the client will reach out to the data node which is closest and start reading the block. Once it is done reading block number 1, it will move on to block number 2 and then block number 3 and so on. - -For a crash scenario, client is reading block number 3 and the data node it is trying to read from is not responding. The client will take a note of the data node which is not responding, it will not try to reach the same data node again for the current read operation. And it will move on to the next data node that has the copy of the block from the list sent by the name node. Therefore, even when a data node is down during the read operation, the read operation continued to progress without any issues. - -### Write operation - -![](assets/hadoop-distributed-file-system-hdfs_read-operation.png) - -Now, we want to write a file to HDFS using `hadoop fs -copyFromLocal` command. Behind the scenes, when a client request a write operation, it will request the name node to allocate blocks for the file and the list of data nodes for each block where the replicas for each block needs to be stored. The name node will do few checks to make sure whether the user requesting the right operation has proper permission to do so and whether the file name already exists in the directory etc. When all the checks are okay, it will proceed with the block allocation. The name node will now have to come up with a list of data nodes. While picking the data nodes to store the replica, the name node will pick the data nodes which are not busy and has enough space to hold the blocks. - -#### Replica placement - -![](assets/hadoop-distributed-file-system-hdfs_replica-placement.png) - -To pick the data nodes to store the replicas for each block, the name node will use a replica placement strategy. The default replica placement strategy will work like this: - -- The first replica will be written on the same node as the client requesting the right. -- The second replica will be placed on a random data node on a different rack from the first one. -- The third replica will be placed on a data node which is chosen at random on the same rack as the second one. - -When the nodes are selected, a data pipeline, as shown in the above image, is formed and the data for the block will be written in the form of packets. The first node will then store the packet and pass it on to the second node. And once the packet is stored on the second node it will be passed on to the third node. Once all three nodes stored the packet, an acknowledgement will be sent back indicating a successful write of the packet. once all the packets for the blocks are written the write operation will move on to the next block. - -![](assets/hadoop-distributed-file-system-hdfs_write-operation.png) - -For example, the block we are writing is named Block 123. While writing Block 123 on Node 2, Node 2 suddenly went down in the middle of the write operation. How do we handle this failure? First the packets for Block 123 are moved to the front so that Node 3 will not miss those packets. Next, the block name will be changed. If we keep the block name as Block 123 when Node 2 finally recovers, it will claim that it has Block 123. But the Block 123 is not complete since Node 2 went down in the middle of the write operation. Assuming that block name will be changed to Block 456. Hence, when Node 2 recovers and claims that it has Block 123, the name node will know that Block 123 is non-existent and will order Node 2 to remove Block 123. The parameter `dfs.namenode.replication.min` indicates the minimum number of replications that is needed for a block, by default, it is one. In case of such failure, the write operation will succeed as long as the block is returned to at least one node. Later, the name node will coordinate with the data nodes and arrange to write the missing replicas. - -## References - -- [HDFS architecture](http://svn.apache.org/repos/asf/hadoop/common/tags/release-0.19.2/docs/hdfs_design.pdf) -- https://hadoop.apache.org/ -- https://en.wikipedia.org/wiki/Apache_Hadoop -- [Hadoop: the definitive guide: storage and analysis at internet scale](https://www.amazon.com/Hadoop-Definitive-Storage-Analysis-Internet/dp/1491901632/ref=sr_1_2?crid=2LTQHKE9WNBNC&keywords=Hadoop&qid=1657604708&sprefix=hadoop%2Caps%2C127&sr=8-2) diff --git a/topics/data/hive-window-and-analytic-functions.md b/topics/data/hive-window-and-analytic-functions.md deleted file mode 100644 index da3ec2975..000000000 --- a/topics/data/hive-window-and-analytic-functions.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -title: Hive window and analytic functions -description: Learn how to use Hive window and analytical functions to calculate complex metrics like a 10-day moving average on stock data with easy-to-understand SQL examples and syntax. -date: 2022-12-12 -github_id: dudaka -icy: 10 -redirect: - - /kJrkXw ---- - -If you're a SQL or PL/SQL developer or developed an ETL process before, you know just plain SQL is not going to get the job done. To implement complex use cases, we need powerful window and analytical functions. The good news is Hive supports both window and analytical functions. Before we look at an example of window and analytical functions, let's understand what they are and why do we need them. Let's take our very old stocks data set and say we want to calculate average volume by stocks by year. We've done this before already on some previous articles about Apache Hive. And the following query show it looks like in Hive. - -```sql -SELECT - year(ymd), - avg(volume) -FROM stocks -WHERE year(ymd) in ('2000', '2001') -GROUP BY year(ymd); -``` - -![](assets/hive-window-and-analytic-functions_select-result.webp) - -We do a select on the year followed by the average aggregate function on volume and then we say `GROUP BY year(ymd)`. To limit the execution, we limit the records to years 2000 and 2001. The result has two records, one record for each year and the average volume by each year. First, the records were grouped by the columns mentioned in the group by clause and then the aggregate function is applied on each group. The important thing to note is that the result is collapsed, we see one record at the end for each group. In our case, we get one record per year. Let's see how do we do this with analytical functions with the following query: - -```sql -SELECT - year(ymd), - avg(volume) OVER( - PARTITION BY year(ymd) - ) -FROM stocks -WHERE year(ymd) in ('2000', '2001'); -``` - -![](assets/hive-window-and-analytic-functions_select-result-analytic-function.webp) - -In the query, we don't have `GROUP BY` clause instead we have `avg(volume)` followed by over `OVER(PARTITION BY year(ymd))`. `PARTITION BY` works similar to `GROUP BY`. The screenshot show a part of the result with a lot of records. With analytical functions, the average function that we have here is run for each record. First, the data set is partitioned or grouped by on the specified column. We have specified `year` as the partition column, so the records are grouped by year. This is referred to as the window. And next, the analytical function gets executed on each record, the average is still calculated on all the records in the partition or the window, but it is executed for each record. That's not very helpful! If a plan groups by an aggregation is a use case, we wouldn't use analytical or window functions. - -So for what use cases we would really need analytical and window functions? Analytical and window functions become interesting when we consider frames. To explain frames, let's consider to a more appropriate use-case. For example, how can we create a 10-day moving average with our stocks data set? - -![](assets/hive-window-and-analytic-functions_apple-ma-10.webp) - -The above screenshot shows an actual chart for Apple at [TradingView](https://www.tradingview.com/). To draw a 10-day moving average on a chart, we can find an indicator with the same name and add it to the chart. The blue line is a plot of 10-day moving average. Each point in the line is an average of 10 day prior closing prices and all points are connected to create a line. Moving average is one of the most commonly used tool in technical analysis of a stock, it is used to see the support and resistance for a stock. How the moving average is used are not so important for this article. What's interesting is how to calculate the 10-day moving average with Hive. The following query is the one with analytical functions to calculate 10-day moving average: - -```sql -SELECT - ymd, - year(ymd) as year, - exch, - symbol, - volume, - price_close, - avg(price_close) - OVER ( - PARTITION BY symbol - ORDER BY ymd - ROWS BETWEEN 9 PRECEDING and CURRENT ROW - ) AS 10_day_moving_average -FROM stocks -WHERE symbol in ('IBM'); -``` - -First, we are partitioning the data by `symbol`. Next we do an order by date (`ORDER BY ymd`) and order by is important for this use case (because when we take the last 10 closing prices, we need the data to be ordered by date or the moving average calculations will be random and incorrect). Let's explain this in an illustration with the following picture: - -![](assets/hive-window-and-analytic-functions_windows.webp) - -We first create partitions by `symbol` and then order the records within each partition. Note that the order by is not a global ordering. Records inside each partition are ordered, so if we have five symbols in our data set, we are creating five partitions or five windows. The window does not overlap with each other and the records inside each window are ordered by date. - -![](assets/hive-window-and-analytic-functions_frames.webp) - -The `ROWS BETWEEN 9 PRECEDING and CURRENT ROW` clause defines the frames. If we zoom into a window, as shown in the above picture, the frame is calculated for each row. For example, the frame for the record with the date 2000-01-18 is the current row (in red) and the 9 records before that. The frame for the record 2000-01-19 is the current record (in pink) and nine records before that, that is from 2000-01-19 to 2000-01-05. The 10 day average is not calculated on the entire window but on the frame for each record in the window. - -The window and analytical functions are now powerful with frames, each record in the window has its own frame and it is dynamic at runtime. There is an important difference between window and frames: windows are not overlapping but frames could be overlapping. The row clause is optional, i.e. if it is not mentioned, rows between unbounded proceeding and current row is applied behind the scenes by default. This means a frame would include the current row and the rows behind the current row within the window. - -In summary, we understand the need for window and analytical functions. Then, we introduced the basics of window and frames. Finally, we learned the syntax for creating windows and using analytical functions in Hive by calculating a 10-day moving average with stocks data set. diff --git a/topics/data/introduction-to-apache-hive.md b/topics/data/introduction-to-apache-hive.md deleted file mode 100644 index 7e44b84f8..000000000 --- a/topics/data/introduction-to-apache-hive.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: Introduction to Apache Hive -description: Learn how Apache Hive simplifies big data analysis by enabling SQL queries on Hadoop datasets using tables, making MapReduce easier for developers and data analysts. -date: 2022-11-10 -github_id: dudaka -icy: 10 -redirect: - - /2dEn0w ---- - -Welcome to one of the widely used tools in the Hadoop ecosystem! To get the best understanding about this tool, please visit the related articles about MapReduce, Apache Pig. In this article, we're going to see a brief introduction to Hive and we're going to create a Hive table and query a dataset using the newly created Hive table. - -As usual, let's start with the question, when we think of analyzing data, for most of us, a database and a table immediately come to our mind. It's not a surprise because we're so used to visualizing data in a table format that is in a row, columnar fashion. And also, almost all of us are familiar with SQL. Data in Hadoop cluster is represented as files and so far MapReduce programming or Pig doesn't allow us to view the data in a table format. Therefore, it makes sense to have a tool in Hadoop ecosystem to represent the data sets in table structure and run SQL queries against it. Hive does exactly that. - -Hive was developed by Facebook and is now a top level Apache project. Just like Pig, it is also widely used in the industry. With Hive, we can create table structures for our dataset and then write SQL queries to analyze our data set. And because of its ease of use, it has a very short learning curve when compared to MapReduce programming and Apache Pig. Hive takes in a SQL query and converts the query into one or more MapReduce jobs and submits the MapReduce jobs to the cluster. If we recollect, Pig does something very similar too: Pig takes in a Pig script using Pig Latin and convert it into one or more MapReduce jobs and submit the job to the cluster. - -Why do we need to have two tools, Pig and Hive, doing somewhat similar things? First of all, Pig and Hive were created by Yahoo and Facebook respectively, to solve the same problem around the same time.The capabilities of each tool was not fully transparent to both companies at the early stages of development which resulted in the overlap. - -The next question is that do companies use both Pig and Hive at the same time? The answer is yes, we have seen successful Hadoop implementations using both Pig and Hive in the same environment.Here is one such use case, we can use Pig for standard nicely extract, transform and load, that is ETL kind of jobs doing predefined aggregation data, cleanup, filtering and structuring of the data, etc. And Hive can used by developers, data analysts and scientists on a day to day basis for ad hoc analysts of data. Just like Pig, Hive is also a client tool, meaning we don't need to have Hive installed on all the nodes in our cluster, it must be installed on nodes where the Hive queries will be initiated from and that node should have access to the Hadoop cluster to execute the MapReduce jobs. That is the comparison between Hive and Pig. - -Now that we know Hive has the concept of tables and queries. Let's look how does Hive compare to the traditional RDBMS systems. Hive leverages Hadoop for processing and hence can process huge volume of data. That is the strength of Hive, but Hive is a batch processing tool because it executes MapReduce jobs in Hadoop cluster behind the scenes. So we will not find all the bells and whistles of a traditional RDBMS system in Hive. Here are some of the key distinctions when comparing Hive and a traditional RDBMS system: - -- We will not find pointed updates or deletes in Hive. -- Hive has limited support for indexing -- very high level transaction support which was introduced very recently. -- There are no support for triggers in Hives - -With these key distinctions, Hive is not a database system rather than a tool that helps developers write MapReduce jobs in Hadoop cluster using simple SQL queries. - -In the Hadoop ecosystem, Hive was a powerful tool to start with: - -- It makes MapReduce very easy. We don't need to know a programming language to write MapReduce programs anymore. All we need to know is SQL and we feel right at home when working with Hive. -- It is easy to implement joints using Hive, again, the query syntax for joints is just as same in a traditional database system. -- Hive helps in structuring the data efficiently with the help of partitions and buckets, which helps an optimized execution of queries. -- Hive supports not just stacks, but different file formats. -- This feature is very important when compared to database: Hive does not enforce the schema on the data that is stored behind the Hive table. For example, if we have a 10-column table in a traditional database system, when inserting a record to the table, we have to supply values for all 10 columns. This means that the traditional database system enforces the schema during the write operation to the table. In a big data environment, our data may not always confront to a structure and Hive understands that. So Hive does not care about the structure of the data and does not enforce the schema on write, whereas it only looks up the schema during the read operation which helps us to work with data which are not fully structured. -- Finally, for any developer who wants to work in Hadoop, Hive is not an optional tool, it is a must-know tool in the Hadoop ecosystem. We will learn all the concepts in detail in next several articles. - -Before we look at all the details and all the important concepts, we start simple. Let's create a database and then create a Hive table and query our stocks dataset. Here is the syntax to create a database and a table. - -```sql -hive> CREATE DATABASE stocks_db; - -hive> USE stocks_db; - -hive> CREATE EXTERNAL TABLE IF NOT EXISTS stocks_tb ( -exch STRING, -symbol STRING, -ymd STRING, -price_open FLOAT, -price_high FLOAT, -price_low FLOAT, -price_close FLOAT, -volume INT, -price_adj_close FLOAT) -ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' -LOCATION '/user/dungho/input/stocks'; - -hive> SELECT * FROM stocks_tb -LIMIT 100; -``` - -If we've worked with any databases like MySQL, SQLServer or Oracle, etc., this syntax will be very familiar to us. To create a database, all we have to say is `CREATE DATABASE` and specify a database name. The database is named `stocks_db`. Then to create objects under the database, we need to switch to the database. So we will say `USE stocks_db`. Next is the syntax to create the table, we're creating a table with name, `stocks_tb` by specifying a list of columns along with its data types. As we can also observe that the data types does look like data types from Java. There are a couple of new elements in the syntax which we will not find in the regular table creation syntax when working with regular databases like MySQL or Oracle. Our stocks data set is a comma delimited data set. So `ROW FORMAT DELIMITED FIELDS TERMINATED BY ','` indicates that the columns in the data set is separated by comma. The `LOCATION` attribute points to the location of the data set in HDFS. And that is where the stocks dataset resides. Each row in the data set has information like opening, closing, high, low prices and volume for a stock for a given day. The other thing is that the table is marked external. We'll look at table types in more detail in next articles. We can query the table is just like we do in MySQL or Oracle, etc. For example, - -```sql -SELECT * FROM stocks_db -LIMIT 100; -``` - -## References - -- https://cwiki.apache.org/confluence/display/Hive//LanguageManual -- The Ultimate Guide To Programming Apache Hive: A Reference Guide Document – Straight from the trenches, with real world lessons, tips and tricks included to help you start analyzing BigData, Fru Nde diff --git a/topics/data/introduction-to-apache-pig.md b/topics/data/introduction-to-apache-pig.md deleted file mode 100644 index b53126493..000000000 --- a/topics/data/introduction-to-apache-pig.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -title: Introduction to Apache Pig -description: Apache Pig simplifies Hadoop data processing by converting easy-to-write Pig Latin scripts into MapReduce jobs, enabling non-programmers to handle big data without coding in Java or Python. -date: 2022-11-14 -github_id: dudaka -icy: 10 -redirect: - - /FGFsgw ---- - -In short, Apache Pig takes a set of instructions from the user and converts those instructions into MapReduce jobs and execute the MapReduce jobs in Hadoop cluster. - -Let's start by asking what's wrong with writing a MapReduce program and why do we need a tool like Apache Pig to translate our instructions into MapReduce jobs. -Here is the answer, there are some challenges with MapReduce programming: - -- First challenge, the ability to conceptually visualize the problem in MapReduce. - This is a problem with most of us, we are so used to our traditional programming approach and now we are introduced to a "Whole New World" of MapReduce, which requires us to think of the solution to a problem in MapReduce sense. - This does not come natural to us. - For example, given a file with row by row employee details like employee name, employee ID, department ID, salary, etc. - Think of how we would calculate the average salary by department in MapReduce. - It's sure we will eventually figure it out but it needs a little bit of effort. - That's our first challenge: conceptually visualizing a problem in MapReduce. -- Second challenge, knowledge of a programming language like Java, C++, Python, etc. - This could be a biggest challenge for many people. - We could be an excellent database developer or a super data analyst, but with traditional MapReduce programming, if we don't know a programming language, we're out of luck. - Question: Is it possible to involve in Hadoop without learning a programming language? - The answer is, of course, yes. - Tools like Pig or Hive to the rescue. -- Third challenge, programming MapReduce in Java, for instance, takes up a lot of time and effort to do simple stuff like Joins for example. - If we are dealing with data, join operations are the most rudimentary operations we would expect to do on our data sets on a regular basis. - But joins are very difficult and time consuming to implement in MapReduce. - Again, Pig is the rescue, with a simple one-line instruction in Apache Pig, we can perform joins. - Pig in the background will do the heavy lifting for us by writing the needed MapReduce jobs and execute them in the Hadoop cluster. -- Last challenge, time and effort with all the challenges discussed about, it's very clear that writing a MapReduce program from scratch with the programming language will require time and effort. - Pig will solve this problem. - As a user, we will provide a set of instructions that Pig understands and Pig will generate one or more MapReduce jobs for us and execute the same in the Hadoop cluster. - Thereby avoiding the need to write even a single line of MapReduce code sounds very promising. - -![](assets/introduction-to-apache-pig_problem-template.webp) - -Question: how a tool can replace the need for programmer and programming? Most of our data problems will follow a problem template, as shown in the above diagram, we will load the data, then filter the data. -It would be for removing bad records or removing some records like employees with salary greater than hundred thousand. -Then, we would perform some grouping of data, that is grouping on one or more columns. -After grouping, we would most likely perform aggregation like average, finding minimum values, maximum values from the group result set. -Finally, we would display or store the result set. -Of course, this template does not show operations like joins, etc. and our problem can be more complicated than this but it shows the idea. -Most of the data problems can be broken down into list of operations and Pig provides instructions for each operation. -At runtime, Pig will take these set of instructions, analyze them and translate them into one or more MapReduce jobs and execute them in Hadoop cluster. - -A little bit of background of Apache Pig: - -- Apache pig is developed at Yahoo. As memtioned in an article about Apache Hadoop, Hadoop was initially funded by Yahoo. - When Yahoo had a successful Hadoop implementation, the need for non-programmers like data scientists, database developers, testers to use the Hadoop platform became more obvious. - So, the Yahoo research team was tasked to create a tool that would help non-programmers to use Hadoop platform. -- Pig's first release came out in September 2008. Pig is not an acronym. - When people at Yahoo were trying to come up with the name, one of the developers suggested Pig and the name got stuck because it was short and sweet. -- Pig is a client. Meaning, we don't have to install Pig in all the nodes in Hadoop cluster. - Pig installation comes with a data flow language called Pig Latin, which defines the instructions that user will use to work with the data. -- The instructions will then be analyzed by an engine and translated into MapReduce jobs. - These MapReduce jobs are then submitted to our Hadoop cluster. - As long as we have Pig installed in one of the nodes in our cluster or installed on a node which has access to the cluster, we're good to go because for Hadoop cluster, a MapReduce job is a MapReduce job whether it was created by a user or created by an external tool like Pig. -- Finally, Pig uses HDFS and MapReduce programming model behind the scenes. - The MapReduce jobs created by Pig will follow the MapReduce phases, which are described in MapReduce's articles. - The tool is merely an enabler for us to execute MapReduce jobs without having to create MapReduce jobs. - -Let's take a look about Pig Latin: - -- Pig Latin is a simple to use data flow language. -- As a user, we typically write a series of instructions using Pig Latin. - For example, if we want to load the data, we would use the load operator. - To filter the data, we would use filter operator. - To group the data, we will use group operator. - Pig Latin also comes with aggregate functions like average for calculating average, min and max functions to calculate minimum and maximum values from a range of values, etc. -- When we execute the Pig instructions, Pig will analyze and optimize the instructions before translating the instructions into MapReduce jobs. - In other words, Pig can do some optimizations to our instructions. - For instance, if we're filtering records, Pig will see whether the filter operator can be moved any higher in the execution chain without possibly affecting the end result. - Because the more sooner we filter the data in our execution chain, the less data subsequent steps will have to process which will definitely result in performance improvement. - So, Pig can help us with optimization as well to an extent along with writing MapReduce jobs for us. - -When Yahoo engineers started to work in developing Apache Pig, they had four philosophies in mind which they thought pigs should adhere to. These philosophies sure sounds funny but it gives a good Insight on what the tool can do: - -- Philosophy 1, pigs eat anything. - Pig can work with data even when we don't specify the metadata or schema or structure of the data set. - Even when we don't specify the column names and its data types, Pig will try to work with the data. We'll find out more about the instructions in next Apache Pig's articles. - But the point here is with very limited instructions, we can understand and process the data and also the data doesn't have to follow a strict schema. - In a big data world we will be getting our data from multiple data points and not always we can expect our data will be structured. - Pig is well suited for unstructured or semi-structured data sets as well. - Also Pig is very forgiving when all the data in our data set does not adhere to a strict schema. -- Philosophy 2, pigs fly. - Pig is built ground up with big data performance requirements in mind. - Pig has an optimizer that could rearrange operators to optimize performance. - Also new requirements and enhancements are made to Pig with performance considerations in mind. -- Philosophy 3, pigs are domestic animals. - Pig is highly configurable. - Pig allows us to write user-defined functions in Java and easily allow us to integrate the code so we're not stuck with just the functions and operators supplied by Pig. - We can write our own code as well. -- Philosophy 4, pigs live anywhere. - Pig is intended to be a language for parallel data processing. - It is not tied to one particular framework like Hadoop. - So far it is very successful with Hadoop, we have to wait and see how this philosophy is going to shine in the future. - -So those are the philosophies, we have to keep in mind when we think of Apache Pig and if we're hoping for a carrier in Hadoop. -Pig is a must know tool and it's very simple and easy to learn. - -In summary, Apache Pig helps to create MapReduce job with ease in Hadoop. - -## Reference - -- https://cwiki.apache.org/confluence/display/pig/ -- Programming Pig: Dataflow Scripting with Hadoop 2nd Edition diff --git a/topics/data/introduction-to-crdt.md b/topics/data/introduction-to-crdt.md deleted file mode 100644 index 852b8c6e8..000000000 --- a/topics/data/introduction-to-crdt.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Introduction to CRDT -description: Conflict-free Replicated Data Types (CRDTs) (aka convergent replicated data type or commutative replicated data type) are data structures that can be replicated across multiple computers in a network, where the replicas can be updated independently and concurrently without coordination between them, and enable operations to always converge to a final state consistent among all replicas. -date: 2023-11-05 -authors: - - fuatto -github_id: fuatto -icy: 10 -tags: - - crdt - - data-types - - distributed-system - - data-structures -redirect: - - /qXYdhA ---- - -## What is CRDT? - -Conflict-free Replicated Data Types (CRDTs) (aka convergent replicated data type or commutative replicated data type) are data structures that can be replicated across multiple computers in a network, where the replicas can be updated independently and concurrently without coordination between them, and enable operations to always converge to a final state consistent among all replicas. - -![](assets/introduction-to-crdt_b41416acdcc17fd248cefbd2f333fb6e_md5.webp) - -## Why CRDT? - -### Addressing data modification dilemma - -There are two approaches to handling concurrent modifications in distributed systems: - -1. **Strongly consistent replication**: In this approach, replicas coordinate for modifications to ensure strong consistency. However, this coordination sacrifices performance and can be limited by the Cap Theorem. The CAP theorem states that in a distributed system, it's impossible to simultaneously guarantee consistency (every read receives the most recent write), availability (every request receives a response), and partition tolerance (the system continues to operate despite network partitions). -2. **Optimistic replication**: In optimistic replication, users can independently modify data without coordination, which enhances performance. However, this approach may lead to conflicts when multiple replicas receive conflicting modifications. To address conflicts, replicas communicate and resolve conflicts automatically. - -### Role of CRDTs - -CRDTs (Conflict-free Replicated Data Types) play a crucial role in **optimistic replication systems**. They enable seamless merging of data modifications by performing replication as commutative operations. CRDTs ensure that conflicting modifications from different replicas are resolved automatically, eliminating the need for special conflict resolution code or user intervention. This makes CRDTs a valuable tool for maintaining consistency in distributed systems while maximizing performance. - -## How to use CRDT? - -CRDTs (Conflict-free Replicated Data Types) are a powerful tool for ensuring data consistency and synchronization in various applications. Here are some examples of how CRDTs can be applied: - -**Applications of CRDTs** - -- _Mobile Apps_: CRDTs enable seamless synchronization of data across multiple devices used by a single user. This ensures that the user can access and modify their data from any device without conflicts or inconsistencies. -- _Distributed Databases_: CRDTs play a crucial role in maintaining data integrity in distributed databases. They allow replicas of the database to be located in different locations while still ensuring that updates and modifications to the data are propagated correctly across all replicas. -- _Collaboration Software_: CRDTs are particularly useful in collaboration software where multiple users can simultaneously make changes to the same file or data. By using CRDTs, conflicts between concurrent changes can be automatically resolved without the need for centralized coordination or locking mechanisms. -- _Large-scale Data Storage Systems_: CRDTs are instrumental in building large-scale data storage systems that require global scalability. By replicating data using CRDTs, these systems can handle high volumes of data and distribute it across multiple nodes, allowing for efficient and reliable access to the data. - -**Decentralized Operation** - -- One of the key advantages of CRDTs is that they support decentralized operation. Unlike traditional systems that rely on a single server, CRDTs can thrive in peer-to-peer networks and other decentralized settings. This makes them well-suited for applications and environments where a central authority is not desirable or feasible. - -## Conclusion - -In short, Conflict-free Replicated Data Types (CRDTs) play a pivotal role in managing conflict resolution. It guarantees the seamless merging of data into a coherent state, irrespective of the modifications made on distinct replicas. Importantly, CRDTs automate this merging process, eliminating the necessity for specialized conflict resolution code or user intervention. - -Unlike systems relying on algorithms utilized by platforms such as Google Docs, Trello, and Figma, CRDTs don't presuppose the reliance on a single server. This distinctive quality allows CRDTs to seamlessly integrate into peer-to-peer networks and various decentralized settings, setting them apart in the landscape of distributed data management. - -## References - -- https://crdt.tech/ -- https://redis.com/blog/diving-into-crdts/ -- https://jakelazaroff.com/words/an-interactive-intro-to-crdts/#user-content-fn-cvrdt -- https://www.infoq.com/presentations/crdt-production/ diff --git a/topics/data/managed-table-vs-external-table.md b/topics/data/managed-table-vs-external-table.md deleted file mode 100644 index e5074ee0f..000000000 --- a/topics/data/managed-table-vs-external-table.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Managed table vs external table -description: Learn the key differences between Hive managed tables and external tables, including when to use each type and how dropping tables affects their underlying data in HDFS. -date: 2022-11-16 -github_id: dudaka -icy: 10 -redirect: - - /ci6C5LU ---- - -In this article, we're going to take a look to two different types of Hive tables and the significance of each. There are two types of tables in Hive: Managed table and External table. Managed table has full control over its data, i.e. when we drop the table, the tables, data set or files will be also deleted from HDFS. External table, however, does not have full control over its data set, i.e. when we drop the table, the data set is not deleted from HDFS. - -Now, the above explanation brings up a very important question: when do we use managed table? And when do we use external table? We would choose to use managed table when Hive is the only application using the data set, whereas we would choose to use external table when the underlying data set pointed by Hive table is shared by many applications like Pig, MapReduce jobs, etc. When multiple applications are interested in a data set, would we keep multiple copies of the same data set one for each application? No, we wouldn't because most likely our data set will be in the magnitude of gigabytes or terabytes and so it does not make sense to keep multiple copies of the data set. That means when a single copy of the data set is shared between application, we don't want Hive to delete the data set when the table is dropped. - -Now, we understand the difference between managed table and external table. Let's see how to create a managed table and how to create an external table. By default, when we create a table, it is a managed table. If we want to create an external table, we have to specify the keyword `EXTERNAL` when creating the table. - -![](assets/managed-table-vs-external-table_managed-table.webp) - -Assume that the stocks table from "Behind a Hive table" article is already on our cluster. So, we can describe formatted on the stocks table, `DESCRIBE FORMATTED stocks_db.stocks` and check out the table type on the output screen shown in the above screenshot. It says `MANAGED_TABLE`, which means our stocks table under `stocks_db` database is a managed table. - -![](assets/managed-table-vs-external-table_managed-table-check.webp) - -Let's check out the data set under the location attribute with the command `!hadoop fs -ls /user/hive/warehouse/stocks_db.db/stocks;`. We can see the data set under the location specified in the location attribute in HDFS. Now what we're going to do is we're going to drop the table and check the location again let's drop the table using the drop table command `drop table stocks_db.stocks;`. The table is now dropped now and check the location again. The above screenshot shows `No such file or directory`, which means that data set is now dropped as well. If anyone tries referring to this data set, it fails since we deleted the data set. That is the behavior of managed table. - -Let's now look at the external table. The following command is the create table syntax to create an external table. The `EXTERNAL` keyword is mentioned in the syntax. - -```sql -CREATE EXTERNAL TABLE IF NOT EXISTS stocks_ext ( - exch string, - symbol string, - ymd string, - price_open float, - price_high float, - price_low float, - price_close float, - volume int, - price_adj_close float -) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; -``` - -Let's execute the command and do a described formatted command on this table. - -![](assets/managed-table-vs-external-table_external-table.webp) - -As shown in the screenshot, the table type is mentioned as external table. Next, we can load this table using this load instruction `LOAD DATA INPATH 'input/hive/stocks_db' INTO TABLE stocks_ext;`. The table is now loaded, let's verify the location of this table before, `!hadoop fs -ls /user/hive/warehouse/stocks_ext/stocks;` and after dropping the table, `DROP TABLE stocks_ext;`. Now that the table is dropped and let's do a listing on the location again. The screenshot show that the data set which is exactly what we expect to see with external table. - -Knowing when to use managed table and when to use external table is crucial. Question: Does using the location attribute when creating a table change the behavior of manage table or external table? The answer is no. When creating a table in Hive, by default, Hive creates a directory for the table under Hive's warehouse directory. For some reason, we don't want the tables directory to be under the warehouse directory, we can override the location using the location attribute during table creation. Another scenario, whereas a Pig script runs every night in our cluster and creates a data set in a HDFS location. Now we want Hive to use this data set. In this case, we would use an external table and use the location attribute to point to the location which is populated by Pig. In this scenario, we could have used managed table as well but external table is more appropriate because this location is also being used by the Pig script to populate the data set. Therefore, we won't drop the location when we decide to drop the table since the location is also being shared by Pig. - -In summary, there are two types of tables in Hive: managed table and external table. When a manage table is dropped, the underlying data will also be dropped. But dropping an external table doesn't drop the data set. Thus, the external table is a good choice when the Hive table is pointing to a data set which is shared by other applications. diff --git a/topics/data/mapreduce-components.md b/topics/data/mapreduce-components.md deleted file mode 100644 index cf1ffae11..000000000 --- a/topics/data/mapreduce-components.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: Mapreduce components -description: Learn how MapReduce processes large stock data to find maximum closing prices using map, shuffle, reduce phases, input splits, and optional combiners for efficient distributed computing and performance. -date: 2022-10-24 -github_id: dudaka -redirect: - - /jOA1wMc ---- - -## Introduction - -[[MapReduce]] consists of four components: - -- Map phase -- Reduce phase -- Shuffle phase -- Combiner - -## Problem statement - -![](assets/mapreduce-components_stock-problem.webp) - -Here's a problem we'd like to solve. We have a data set with information about several fictitious stock symbol. In each line in the data set, we have information about a stock symbol for a day: opening price, closing price, high, low, volume, etc. - -Let's pick the first line in the above picture, it is going to be our record. The first is the exchange name, `ABCSE`, ABC stock exchange. The next is the symbol - `B7J`, the date - `2008-10-28`, the opening price - `6.48`, closing price - `6.72`, high - `6.74`, low - `6.22` for the day and the volume - `44300`. This data set is about 400 MB, not too big but good enough for our experimentation and learning. - -Now, it is about the problem we would like to solve with this data set. For every stock symbol in the data set, we would like to find out its maximum closing price across several days. - -![](assets/mapreduce-components_stock-algorithm.webp) - -The above diagram shows the algorithm. We'll read a line, get the symbol on closing price from the line, then we need to check if the closing price is greater than the closing price we have for that symbol. - -If not, go to the next line. Otherwise, the closing price is greater than the closing price that you already have for that symbol, save the closing price as the maximum closing price for that symbol and move on to the next record in the data set. - -If the end of the file is reached, print the results. - -The problem with this approach is that there is no parallelization. Thus, if we have a huge data set, we will have extremely long computation time which is not ideal. - -### Input split - -![](assets/mapreduce-components_distributed.webp) - -Let's consider how we have worked out the same problem in the mapreduce world. From the article `What MapReduce is`, we got introduced to the faces of MapReduce, so we'll take this problem and go over each phase and see the technical details involved in the map phase, reduce phase and shuffle phase. - -First, it is about the map phase, the central idea behind MapReduce is distributed processing. So, first thing is to divide the data set into chunks and you have separate process working on each chunk of data. The chunks are called input splits and the process working on the chunks are called mappers, as show as in the above picture. Each mapper would process a record at a time and each mapper would execute the same set of code on every single record. -The output of the mapper would be a key-value pair. - -![](assets/mapreduce-components_input-splits-vs-blocks.webp) - -Is it true that input split is same as the block? Input split is not same as the block. -A block is a hard division of data at the block size. If the block size in your cluster is 128 MB. Each block for the data set will be 128 MB except for the last block which could be less than the block size if the file size is not entirely divisible by the block size. Since a block is a hard cut at the block size, a block can end even before a record ends. -In the above diagram, we have four records in our data set and each record is 100 MB and the block size of our cluster is 128 MB. So the first record will perfectly fit in the block since the record size is 100 mb it's built within the block size which is 128 MB. However, the second record cannot fit in the block, so the record number 2 will start in block 1 and will end in block 2. - -If we assign a mapper to block 1, in this case, the mapper cannot process record 2 because block 1 does not have the complete record 2. This is exactly the problem input split solves. In this case, input split 1 will have both record 1 and record 2. -Input split 2, however, does not start with record 2. - -Since record 2 is already included in the input split 1. so input split 2 will have only record 3. Record three is divided between block 2 and block 3. Input split is not physical chunks of data, it is a Java class behind the scenes with pointers to start and end location within blocks. - -Therefore, when a mapper tries to read the data, it clearly knows where to start and where to end. The start location of an input split can start in a block and can end in another block. - -So that is why we have a concept of input split. Input split respects logical record boundary. During mapreduce execution hadoop scans through the blocks and create input splits which respects record boundaries. - -### Map phase - -![](assets/mapreduce-components_map-phase.webp) - -With the understanding about input splits, we can take a look about the mapper in detail. A mapper in hadoop can be written in many different programming languages, it can be written in C++, python Scala and Java. In our case we'll look at Java, a mapper is a Java program in our case which is invoked by the Hadoop framework once per every record in the input split. - -So if you have 100 records in a input split, the mapper processing the split will be executed 100 times. - -**\*Question**: how many mappers will Hadoop create to process a data set? -Answer: the number of mappers is entirely dependent on the number of input splits.\* - -If there are 10 input splits, there will be 10 mappers. -If there are 100 input splits, there will be 100 mappers - -So a mapper is invoked for every single record in the input split and then the output of the mapper should be a key value pair. In our sample stock data set, every line is a record for us and we need to parse the record to get the stock symbol and the closing price. The stock symbol and the closing price becomes the output from each execution of the mapper: the symbol is going to be the key and the closing price is going to be the value in your key value pair. - -But how do we decide what should be the key and what should be the value in our key value pair? - -### Reduce phase - -![](assets/mapreduce-components_reduce-phase.webp) - -The reduce phase that will give us an answer. The reducers work on the output of the mappers. The output of individual mappers are grouped by the key, in our case, the stock symbol and pass to the reducer. Reducer will receive a key and a list of values for that key for input. The keys will be grouped. - -For example, our data set has stock information about 10 stock symbols and 100 records for each symbol so that is 1000 records in total, 10 stock symbols and 100 records for each stock symbols that is thousand records. - -We will get 1000 key value pairs from all mappers combined because our mapper will be executed for each record When processing a record, we can decide not to output a key value pair for the record. For instance, the record could be bad, in that case, we won't output a record from the mapper. - -But in an ideal scenario, we will have 1000 key value pairs because we have thousand records then the reducer will receive 10 records to process. One record for each symbol since we only have information about 10 stocks. Each record for the reducer will have a symbol for the key and a list of closing prices for value that is all we need to calculate the maximum closing price for each symbol. - -The work of the reducer becomes simple, it reads the key and calculate the maximum closing price from the list of closing prices for that symbol and output the result. - -**\*Question**: how do we decide what should be the key and what should be the value?\* - -There is a simple trick: think about what needs to be reduced. In our example, we know if the reducer has the stock symbol and the list of closing prices for a given stock symbol, we can arrive at the maximum closing price and also we want the reducer to be called once per symbol that is why we made symbol as the key in mapper's output and closing price as the value. - -We know the number of mappers equals to the number of input splits are not controlled by the users. Number of reducers can be set by the user, we can even have a map reduced job with no reducers. - -Assuming that data set is divided into 100 splits which means 100 mappers. Now we have only one reducer to process all the output from 100 mappers. In some cases it might be okay but we might run into performance bottleneck at the reduced phase because we're trying to reduce output from 100 mappers in one reducer. So if we're dealing with large amount of data in the reduced phase it is advisable to have more than one reducer. - -### Shuffle phase - -![](assets/mapreduce-components_multiple-reducers.webp) - -In the above picture, we have multiple reducers. Let's consider how the output of the individual mappers got grouped by symbols and reached the reducer. The magic happens in the shuffle phase. - -Shuffle phase is also a key component in mapreduce. The process, in which the map output is transferred to the reducers, is known as a shuffle. Let's take a look at the shuffle phase in detail. - -Assuming that in our mapreduce job, we decided to use three reducers. For example, we have have data for Apple in the stock data set and we have 10 input splits to process which means we will need 10 mappers. - -We can have records for Apple in more than one input split. Let's say the records for Apple is spread out in all the 10 input splits, this means each mapper will produce key value pairs for Apple in its output. - -When we have more than one reducer, we don't want the key value pairs for Apple to be spread out between the three reducers that will be bad for our use case because we won't be able to calculate the consolidate max closing price for Apple. -Therefore, we want all the key value pairs for Apple to go to one reducer. -In other words we want each key or symbol in our case to be assigned to a reducer and stick with it. - -In the map phase, each key is assigned to a partition. So if we have three reducers, we will have three partitions and each key is assigned to a partition by a class called partitioner. - -If the partitioner decides that any key value pair with Apple as key should go to partition 1 then all key value pairs with Apple as key will go to partition 1 and each partition will be assigned to a reducer: Partition 1 will be assigned to Reducer 1, Partition 2 will be assigned to Reducer 2, etc. It is key to understand that this partitioning happens across all the mappers in the map phase. - -Hadoop framework will guarantee that input to the reducers is sorted by key and so once the keys are assigned to the right partition the key value pairs in the partition are sorted by key. - -Once the keys are sorted, we are now ready to copy each partition to the appropriate reducers. This is known as the copy phase, we have to understand that data for partition 1, for instance, can come from many mappers because in our example the records for Apple can be spread across multiple input splits. Therefore, in the reduced phase, the partitions have to be merged together maintaining the sort ordering by key even though the intense sorting happened at the map phase. - -In some documentation, we will see the merge action referred to as sort on the reduce side. Once the reducers have received all the partitions from all the mappers and the partitions are merged, the reducer will perform the actual reduce operation. -That's the shuffle phase. - -Let's summarize the shuffle phase. Each mapper will process all the records in its assigned input split and will output a key value pair for each record. If we look at the output, we have symbol for key and closing price as value. For example, in the above picture, we can see here `ABC` is a symbol and `60` is the closing price for `ABC`. -Similarly for symbol `STT`, we have closing price as `82`. - -Same for other mappers as well, we may also note that symbols in mapper 1 can also be found in mapper 2. Look at the symbol `STT` for instance, we have `STT` in mapper 1 and we can also see `STT` in mapper 2. Then in the shuffle phase within each mapper the key value pairs will be assigned to a partition. - -Within each partition the key value pairs will be sorted by key. As shown as in the above picture, the output key value pairs are nicely sorted by key in each mapper. Then, the key value pairs from each mapper will be copied over to the reduced phase to the appropriate reducers. - -At each reducer the key value pairs coming from different mappers will be merged maintaining the sort order. - -There are two things to note in the picture: - -- the symbols are unique to each reducer meaning even though records from symbol were widespread across multiple mappers they were sent to one reducer. Take a look at symbol `ABC` for instance, `ABC` was found in mapper 1 and `ABC` was also found in mapper 2 but key value pairs for symbol `ABC` is sent to only one reducer, in this case, reducer 1. - Similarly you can find key value pairs for symbol `STT` in mapper 1 and also in mapper 2 but the key value pairs for `STT` is sent to only one reducer, in this case, reducer 2. -- Once the key value pairs are copied and merged, the job for reducer is very simple. Reducer 1 will run three times, one for each symbol and reducer 2 will run two times, one for each symbol - -Each run will print the symbol and its maximum closing price. That's the end to end process in mapreduce. - -### Combiner - -![](assets/mapreduce-components_combiner.webp) - -We could also have an optional combiner at the map phase. -Combiners can be used to reduce the amount of data that is sent to the reduce phase. In our example, there is no reason to send all the closing prices for each symbol from each mapper. As shown in the above picture, in mapper 1, we have three records for symbol `ABC`: one record with closing price `60`, one record with closing price `50` and one record with closing price `111`. - -Since we are calculating the maximum closing price, we don't have to send the key value pairs with closing price `50` and `60` because they are less than the closing price `111`. Thus, all we need to do here is we need to send the key value pair with closing price `111` for symbol `ABC` from mapper 1 to the reducer. - -Intuitively, combiner is like a mini reducer that runs at the map phase. Combiners can be very helpful to reduce the load on the reduce side. Since we're reducing the amount of data that are being sent to the reducers, thereby increasing performance. Combiners are optional. - -## Summary - -- the internals of map shuffle and reduced phases. -- the benefit of using a combiner. diff --git a/topics/data/mapreduce.md b/topics/data/mapreduce.md deleted file mode 100644 index 8e6a95eea..000000000 --- a/topics/data/mapreduce.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: ¶ MapReduce -description: We are the head of census bureau for the state of California and tasked with finding the finding the population of all cities in California. All the resources we want are ready but we have only four months to finish the task. Calculating the population of all cities for a big state like california is not an easy task. The sensible thing to do is to divide the state by city and make individuals in charge of each city to calculate the population of each city where he is in charge of... -date: 2022-10-24 -authors: - - dudaka -github_id: dudaka -icy: 10 -tags: - - data - - hadoop - - mapreduce - - distributed -redirect: - - /2KVHtg ---- - -## What is MapReduce? - -Let's start with an example. - -We are the head of census bureau for the state of California and tasked with finding the finding the population of all cities in California. All the resources we want are ready but we have only four months to finish the task. Calculating the population of all cities for a big state like california is not an easy task. The sensible thing to do is to divide the state by city and make individuals in charge of each city to calculate the population of each city where he is in charge of. - -![](assets/mapreduce_mr-example-1.webp) - -For illustration purpose, there are three cities: San Francisco (SFO), San Jose (SJOSE) and LA. - -Person 1 will be in charge for SFO, person 2 will be in charge for San Jose, and person 3 will be in charge of LA. - -We have divided California into cities and each city is assigned to a person and he is responsible for finding the population of the assigned city. We now need to give instructions to each person on what they have to do. Thus, we ask each person to go to a home knock on the door and when someone answers the door, ask how many people live in the home and note it down. Each one notes down the city they're responsible for and the number of people live in the home. Then, the person has to go to the next home and repeat the same process until he covers all homes in the assigned city. - -For a person who covers SFO, he goes to first home, there are five people in the home so he note down `SFO 5`, three people are living in the second home so he note down `SFO 3`, so on. It's a classic divide and conquer approach. -Same instructions will be carried out by everyone involved person 2 will go to San Jose and Person 2 will do the same in LA. When each person is done with their assigned city, we ask them to submit their results to the state's headquarters. -We'll have a person in the headquarters to receive the results from all cities and aggregate them by city to come up with population of each city for the entire state. -Therefore, four months in with this strategy, we're able to calculate the population of california. - -![](assets/mapreduce_mr-example-2.webp) - -Next year, we're asked to do the same job, we have all the resources we want but this time we have two months to finish the task. So we would simply double the number of people to perform the task. We will divide SFO into two divisions and add one person to each division and we will do the same thing for San Jose and LA. Each person responsible for a division will perform the same task as before, we can also do the same thing at the headquarters. - -Let's divide the headquarters into two: `CA HQ 1` and `CA HQ 2` and one person to each division. With twice as much people, we can finish the task in half the time but there is one small problem, we want the census takers for SFO, called `SFO 1` and `SFO 2`, send their results to either `CA HQ 1` or `CA HQ 2`. - -We don't want `SFO 1` sending results to `CA HQ 1` and `SFO 2` sending their results to `CA HQ 2` because this would result an population count for SFO divided between `CA HQ 1` or `CA HQ 2`. That is not ideal because we want consolidated population count by city, not partial counts. - -So, what we can do is to instruct census takers in `SFO 1` and `SFO 2` to send their results to either `CA HQ 1` or `CA HQ 2`. Similarly we should instruct census takers for San Jose and LA, they should either send it to `CA HQ 1` or `CA HQ 2`. -With this model, again, we were able to complete the census calculation in two months. - -If next year if we were asked to do the same thing in a month, we know exactly what to do and we can simply double the resources and apply our model. Now, we have a good enough model, not only the model works but it also can scale. - -The model we have here is called _MapReduce_. MapReduce is a programming model for distributed computing. It's not a programming language, it is a programming model which we can use to process huge data sets in a distributed fashion. - -![](assets/mapreduce_mr-example-3.webp) - -Now let's look at the faces involved in Mapreduce. The phase where individuals collect the population of their assigned city or part of the city is called a [[MapReduce Components#Map Phase|_Map phase]]. - -The individual person involved in the actual calculation is called the _Mapper_ and the city or the part of the city he is working with is known as the _Input Split_. -The output from each mapper is a _Key Value pair_ as we can see the key is `SFO` the value is `5` or the key is `SJSOE` the value is `2`. - -The phase where you aggregate the intermediate results from each city or mappers in the headquarters is called the _Reduced phase_ and the individuals who work in the headquarters are known as _Reducers_ because they reduce or consolidate the output from many different mappers. Each reducer will produce a result set. - -The phase in which the values from the different mappers are copied or transferred to reducers is known as the _Shuffle phase_. The shuffle phase comes in between map and the reduced phase. Therefore, map phase, shuffle phase and reduce face are the three phases of Mapreduce. - -## Summary - -MapReduce is: - -- A distributed programming model for processing large data sets -- Conceived at Google and Hadoop's adapt this programming model. -- Can be implemented in any programming language and Hadoop supports a lot of programming language to write Mapreduce programs. We can write a Mapreduce program in Scala, Python, C/C++ and of course Java. -- Mapreduce is not a programming language, it is a programming model. -- Hadoop implements Mapreduce so that the Mapreduce system in Hadoop manages the communicationsm, data transfer, parallel execution across the distributed servers or nodes. diff --git a/topics/data/order-by-vs-sort-by-vs-distribute-by-vs-cluster-by.md b/topics/data/order-by-vs-sort-by-vs-distribute-by-vs-cluster-by.md deleted file mode 100644 index 7d79ad73a..000000000 --- a/topics/data/order-by-vs-sort-by-vs-distribute-by-vs-cluster-by.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Order by vs sort by vs distribute by vs cluster by -description: Learn how to optimize Hive queries by using ORDER BY, SORT BY, DISTRIBUTE BY, and CLUSTER BY to efficiently order large datasets with multiple reducers and improve performance. -date: 2022-11-23 -github_id: dudaka -icy: 10 -redirect: - - /THOKIQ ---- - -These are very interesting concepts which are about ordering records in a data set. What is so special about ordering? If we want to order the records in the stocks data set by closing price in descending order, we can write a simple query like - -```sql -SELECT * -FROM stocks -ORDER BY price_close DESC; -``` - -However, this simple `ORDER BY` statement has some performance implication. When executed this query, the first thing we will notice in the below output snapshot is `Number of reduce tasks determined at compile time` equals to `1`. - -![](assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot.webp) - -Why Hive is choosing to run the `ORDER BY` statement with just one reducer? Because `ORDER BY` does a global ordering of all records in the data set, which means to do a global ordering all the records in our data set must be sent to one reduce. This is a serious problem if we have a very large data set and when all the records in our data set are sent to one reduce, this will lead to memory issues and the execution time of this reducer could be off the charts. Therefore, the solution is to use multiple reducers instead of just one. - -We can set the number of reducers that we would like to use using the property `mapreduce.job.reduces` in our Hive session. For example, let's set the number of reducers to 3 and run the `ORDER BY` query again. The output shows in the following snapshot. - -![](assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-set-property.webp) - -Again, we are seeing that the number of reduced task is set to 1. Since the `ORDER BY` does global ordering of our data set the number of reducers will be always forced to one even when we specify to more than one reducer. So, what is the real solution here? The answer is `SORT BY`. When using `SORT BY`, it uses multiple reducers. Let's consider the following query: - -```sql -SELECT ymd, symbol, price_close -FROM stocks -WHERE year(ymd) = '2003' -SORT BY symbol ASC, price_close DESC; -``` - -For simplicity, we are filtering only records from year 2003 and we are sorting the records by symbol in ascending order and closing price in descending order. To review the results from this query execution, we are storing the results of the query in the local file system using the following `INSERT` command: - -```sql -INSERT OVERWRITE LOCAL DIRECTORY '/home/dungho/output/hive/stocks' -ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' -SELECT ymd, symbol, price_close -FROM stocks -WHERE year(ymd) = '2003' -SORT BY symbol ASC, price_close DESC; -``` - -We're saying `INSERT OVERWRITE LOCAL DIRECTORY` and we are giving the location in the local file system and we're also saying the output has to be delimited by comma. Now the output of this select statement will be written into this directory delimited by comma. Before we execute this query, let's set the number of reducers to 3. - -![](assets/order-by-vs.-sort-by-vs.-distribute-by-vs.-cluster-by_order-by-output-screenshot-sort-by.webp) - -As shown in the above screenshot, the number of reducers is now set to 3. When the job is complete, the output of this job is copied to the local directory. We can go to the local directory and review the output. There are three files where are one for each reducer. If we open one of these files, we can see the records in this file are sorted by symbol first in ascending order and then sorted by closing price in descending order. - -Unfortunately, there is a problem. Let's pick the symbol `B3B` in the first file, we can find records for `B3B` in this file. And we can also find the records for `B3B` in other files.In this second file as well as the third file, we see the records are sorted by symbol first in ascending order and then sorted by closing price and descending order. In other words, the problem is the symbols from the first file also appearing in other files. They're not duplicates, it is just that the records for the same symbol are distributed between the reducers and then sorted in each reducer. That is not ideal. For true logical ordering, we want all the records from the same symbol to go to the same reducer and end up in one file. - -How do we make all the records from the same symbol go to the same reducer and finally end up in the same file? The answer is `DISTRIBUTE BY` along with `SORT BY`. In the `DISTRIBUTE BY` clause, it specifies the column that should be treated as the key for the reducers. In our case, we would like all the records for the same symbol to go to the same reducer, so we will specify the symbol column in `DISTRIBUTE BY`. The previous query is revised as follows: - -```sql -INSERT OVERWRITE LOCAL DIRECTORY '/home/dungho/output/hive/stocks' -ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' -SELECT ymd, symbol, price_close -FROM stocks -WHERE year(ymd) = '2003' -DISTRIBUTE BY symbol -SORT BY symbol ASC, price_close DESC; -``` - -When executing this query, we can see the number of reducers is set to 3. And we may also notice, since we are using three reducers as opposed to just one, our job is completing much faster. When the job is complete, let's review the output in the output location and there are three files again. If we open the first file, the records are now sorted by symbol in ascending order and then sorted by closing price in descending order as exactly expected. Moreover, we can verify that each symbol is written into only one file. For example, let's pick up the same symbol that we used before `B3B` and make sure that the records for symbol `B3B` is only present in one file, in this case, file number one. Thus, if we go to file number two and file number three and check if whether there are records for `B3B`, technically, we should not see records for `B3B` in any other files since it is already present in file number one because we use `DISTRIBUTE BY` along with `SORT BY`. - -Now the records are not only sorted properly but also do not see overlapping results between files. One last thing, if we have the same set of columns in `SORT BY` and `DISTRIBUTE BY` and we're sorting the records in ascending order, we can replace `SORT BY` and `DISTRIBUTE BY` with `CLUSTER BY`. For example, the following query, in which we have `DISTRIBUTE BY symbol` and `SORT BY symbol`, can be replaced `SORT BY` and `DISTRIBUTE BY` with `CLUSTER BY` as shown in the last query. Both two queries are essentially the same and will give the same output. - -```sql -SELECT ymd, symbol, price_close -FROM stocks -DISTRIBUTE BY symbol -SORT BY symbol ASC; -``` - -```sql -SELECT ymd, symbol, price_close -FROM stocks -CLUSTER BY symbol; -``` - -In summary, `ORDER BY` does global ordering and will always use one reducer, which is problematic because it will lead to performance problems. We can use `SORT BY` along with `DISTRIBUTE BY` to use multiple reducers and send records from a certain key column to the same reducer. Finally, `CLUSTER BY` can be used when the same set of columns are used in `SORT BY` and `DISTRIBUTE BY`. diff --git a/topics/data/overview-of-bi-tools.md b/topics/data/overview-of-bi-tools.md deleted file mode 100644 index 713fb1c80..000000000 --- a/topics/data/overview-of-bi-tools.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Overview of BI tools -description: Compare top data visualization tools like Tableau, Power BI, and Google Data Studio to find the best business intelligence software for interactive dashboards, ease of use, pricing, and integration. -date: 2022-11-21 -github_id: Bachpx2911 -redirect: - - /i7ioAA ---- - -Data visualization is one of most important skill of data analytic. Companies utilize the software for data visualization to have a greater insight into their company data. By using Business Intelligence software, one can do lots of excellent operations with the data, like interact, interpret, and exhibit it. There are several data visualization tools on market such as: Power BI, Sap, Oracle BI,..Power BI , Tableau, Google data studio are the most popular tools. I have chances to work with all of them in my career, so I will review the pros and cons of each tools in this article. - -In regard to Tableau, this platform allows the connection of data from virtually any origin in practically all form. An interactive visual analytics dashboard could be -designed with uncomplicated drag and drop. Most of developer said that Tableau is easy to use and one of the most powerful visualization tool. It can handles huge volume of data, and can be integrated with hundreds of 3rd party applications. However, this price of using Tableau is quite high, approximately 10 times higher than Power BI. - -Other tool is Power BI, which is a set of software for visualizing data. Power BI allows a user-friendly interface and exceptional capabilities for data visualization. If -your business uses Microsoft tools and you have your human resources who are Excel power users, then Power BI at a cheap price and would reasonably serve fine for you. It can be said that this product is built for existing Microsoft users with affordable pricing. However, there are also some cons of Power BI. I think this product need to improve some configuration of Visuals as well as Rigid Formulas - -The last tool in my article is Google Data Studio. Data Studio is a business intelligence tool from Google provided as a completely managed web service. Data Studio provides an intuitive interface to explore and build insights using data. It is built on top of the Google app ecosystem and offers tight integrations to Google based data sources like Big Query, Google Analytics, Google Sheets, etc. Tight integration with Google ecosystem, free and easy to use may be the advantage of Google data studio. However, this app also lack of real-time updates in dashboard and lack of data modelling. Creating a complicated report with huge volume of live connections data may be difficult with Google data studio. - -## References - -- https://improvado.io/blog/review-the-best-data-visualization-tools diff --git a/topics/data/partitions-on-apache-hive.md b/topics/data/partitions-on-apache-hive.md deleted file mode 100644 index 6b0c29b68..000000000 --- a/topics/data/partitions-on-apache-hive.md +++ /dev/null @@ -1,231 +0,0 @@ ---- -title: Partitions on Apache Hive -description: Learn how Hive partitions and dynamic partitions optimize query performance by targeting specific data subsets, reducing scan times, and efficiently managing large datasets for faster Hive queries. -date: 2022-12-02 -github_id: dudaka -icy: 10 -redirect: - - /1Jk3PA ---- - -Have you ever been in a situation where you are trying to optimize a slow running query to make it run faster? In our case, we have been looking at that query for hours and realized that the query is scanning the entire table and we are thinking that this query will be super fast if it only targets specific set of records instead of the entire table. Thus, what we really want, hopefully you too, in such cases is partitions. - -Let's find out about partitions in Hive with the following two parts: - -- Partitions: - - - What are partitions? - - Benefits of partitions - - Creating and loading partitions in Hive. - -- Dynamic partitions - - What are dynamic partitions? - - Benefits of dynamic partitions - -## Partitions - -For example, we want to query the `stocks` table to look at the stock's details for symbol `XYZ` on 2000/07/03. Even though, as a user, we're only interested in one stock symbol for a specific date, this query, however, will run a MapReduce job which will scan the entire data set to get the result set. This means that the execution time will be longer. Wouldn't be so nice if we can Target the query to scan only the records that belong to the symbol `XYZ` to get the result set? There is a way to do exactly that in Hive and it is by using partitions. Now, the `stocks` table has no way of differentiating the records for symbol `XYZ` with records for symbol `ABC`. Using partitions in Hive, we can basically compartmentalize our data set. The syntax for creating a partition table is very similar to a regular table, the only difference is the partition table will have the `PARTITION BY` clause and we have to mention a new name to the partition column. - -```sql -CREATE TABLE IF NOT EXISTS stocks_partition ( - exch STRING, - symbol STRING, - ymd STRING, - price_open FLOAT, - price_high FLOAT, - price_low FLOAT, - price_close FLOAT, - volume INT, - price_adj_close FLOAT -) PARTITIONED BY (sym STRING) -ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; -``` - -In the `stocks_partition` table, we are naming the partition column as `sym` as we want to partition the data set by symbol. Note that this column is not the same column as the `symbol` column in the table. When the table is created, let's execute the describe command on the table: - -```sql -DESCRIBE FORMATTED stocks_partition; -``` - -![](assets/partitions-on-apache-hive_describe_stocks_partition.webp) - -As the above screenshot, the describe information for this table shows partition information the column name for our partition is `sym` and the data type of that is string. To load data into the partition table, it is slightly different from loading a regular table, we're selecting all the records from the `stocks` table with simple `B7J` and inserting those records into the `stocks_partition` table. The important thing to note here is that we're assigning `B7J` to be the value of the partition column `sym`. - -```sql -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym = 'B7J') -SELECT * FROM stocks s -WHERE s.symbol = 'B7J'; -``` - -![](assets/partitions-on-apache-hive_inserting-b7j.webp) - -When the execution of the above command is completed, as shown in the screenshot, Hive created a partition in `stocks_partition` table with the name `sym` equals `B7J`. Assume that another partition for symbol `BB3` is created with the following command: - -```sql -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym = 'BB3') -SELECT * FROM stocks s -WHERE s.symbol = 'BB3'; -``` - -We can list the partitions for a given table with `SHOW PARTITIONS stocks_partition;`, tthere are two partitions. - -![](assets/partitions-on-apache-hive_show-partitions.webp) - -How the data is physically structured for this table `stocks_partition` in HDFS? We can get the value from the location attribute with the command `DESCRIBE FORMATTED stocks_partition;`. - -![](assets/partitions-on-apache-hive_location.webp) - -Usually, we will see files under the tables directory, but the partition tables are structured and stored slightly differently. As shown in the above screenshot, under the directory `stocks_partitions`, we see two more directories one for symbol `B7J` and the other one for symbol `BB3`. These are partition directorie. In the directory `B7J`, there is a file which will have just the records for symbol `B7J` nicely stored in the partition directory. When we query the data for symbol `B7J` using the partition column `sym`, the MapReduce job will only scan this specific directory and that is quite powerful. Since we are not scanning the entire data set anymore, the execution time of this query will be much faster. - -We can also load a partition from a HDFS location. For example, the records for symbol `ZUU` is in the directory `output/hive/stocks-zuu`, which we can load by using this insert command: - -```sql -INSERT OVERWRITE DIRECTORY 'output/hive/stocks-zuu' -SELECT * -FROM stocks WHERE symbol='ZUU'; -``` - -This insert command will create this output directory `stocks-zuu` and it will select all the records from the `stocks` table with symbol `ZUU` and will insert all the records into the newly created directory. Then, we can add a partition to the table using the alter command like this: - -```sql -ALTER TABLE stocks_partition ADD IF NOT EXISTS -PARTITION (sym = 'ZUU') LOCATION 'output/hive/stocks-zuu'; -``` - -The partition is now created for symbol `ZUU`, we can execute `SHOW PARTITIONS stocks_partition;` again to see the list of partitions for the table and there are three partitions created: one for symbol `B7J`, second for symbol `BB3` and finally for symbol `ZUU`. - -We can also create multiple partitions with one insert and multiple selects like this: - -```sql -FROM stocks s -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym = 'GEL') -SELECT * WHERE s.symbol = 'GEL' -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym = 'GEK') -SELECT * WHERE s.symbol = 'GEK'; -``` - -This instruction is very simple to understand but it also looks a little weird. First, we're starting with the table name and then we have an `INSERT` followed by a `SELECT`. Nex, another `INSERT` is followed by a `SELECT`. Here, we will be creating two partitions: one for symbol `GEO` on the other one for symbol `GEK`. For both partitions, we will be selecting appropriate records from the `stocks` table. - -```sql -ALTER TABLE stocks_partition DROP IF EXISTS PARTITION(sym = 'GEL'); -``` - -As mentioned on the article [Managed Table vs External Table](managed-table-vs-external-table.md), we cannot delete records from a Hive table but with partition tables, we can drop partitions using the drop command, which will essentially result in deleting all the records for that partition. For instance, we want to delete all the records for symbol `GEL`. Usually, we will not be able to do that using Hive as we would do with the delete statement in SQL. But with the help of partition, we can drop the entire partition. Here, we're essentially dropping the partition `GEL` which has all the records for symbol `GEL`. - -```sql -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym = 'APPL') -SELECT * FROM stocks s -WHERE s.symbol = 'MSFT'; -``` - -Let's consider the above insert, we are loading records for Microsoft into Apple partition. -This is logically wrong since it will return Microsoft records when a user queries the Apple partition. -Hive will not validate the data that is loaded into partitions and also it will not raise any errors when we incorrectly load data into partitions -It is the developer's responsibility to make sure the partition is loaded with correct set of records. - -## Dynamic partitions - -Imagining that we are trying to create partitions for thousands of symbols. Manually creating partitions one by one is a tedious exercise and also will eventually lead to errors like the one we just saw with Apple and Microsoft. Dynamic partition inserts solve that problem. So far we have been giving the values for the partition columns and this means that the values for the partition columns are known at compile time. When we use Dynamic partition inserts, however, the partition column values are known at execution time. To enable the dynamic partition, we set `hive.exec.dynamic.partition` to `true` with command `SET hive.exec.dynamic.partition=true;`. - -```sql -INSERT OVERWRITE TABLE stocks_partition -PARTITION (sym) -SELECT s.*, s.symbol -FROM stocks s; -``` - -Now, in the above query, we're not hard coding symbols anymore. At runtime, the symbols will be resolved. For each symbol in the `stocks` table, a partition will be created and all the records for that symbol will be loaded into the appropriate partition. - -This is the interesting part! When executing the above insert query, it will give an exception. By default, `hive.exec.dynamic.partition.mode` parameter is set to `strict`, so in strict mode, we would need to have at least one partition column to be given a static value. This is to avoid careless errors by dynamically loading the partitions with, for instance, the `date` column since we'll end up with many number of partitions, one for each date in the dataset. Since we have only one partition column, we cannot load this table in strict mode. We can change the `hive.exec.dynamic.partition.mode` parameter to `non-strict` mode, but then we will leave the table vulnerable to errors. So, `strict` mode is recommeneded. - -```sql -CREATE TABLE IF NOT EXISTS stocks_dynamic_partition ( -exch STRING, -symbol STRING, -ymd STRING, -price_open FLOAT, -price_high FLOAT, -price_low FLOAT, -price_close FLOAT, -volume INT, -price_adj_close FLOAT) -PARTITIONED BY (exch_name STRING, yr STRING, sym STRING) -ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; -``` - -Let's create another table named `stocks_dynamic_partition` with the above query. The table has three partition columns: exchange name `exch_name`, year `yr` and symbol `sym`. - -```sql -INSERT OVERWRITE TABLE stocks_dynamic_partition -PARTITION (exch_name='ABCSE', yr, sym) -SELECT *, year(ymd), symbol -FROM stocks; -``` - -For the table `stocks_dynamic_partition`, with the above insert command, we are giving a static value of `ABCSE` for the first partition column exchange name. In the select statement, the dynamic partition column must be specified last and in the same order in which they appear in the partition clause. We have given a static value for the first partition column, the value for the next two column will be derived from the select statement. Technically, this query should execute but Hive is known to have stability issues when we try to create Dynamic partitions with huge data set. The link that talks about this issue is in References section. The later versions of Hive will fix this issue. So, to avoid this issue let's create partitions for records from 2001 to 2003 with the following insert command. - -```sql -INSERT OVERWRITE TABLE stocks_dynamic_partition -PARTITION (exch_name='ABCSE', yr, sym) -SELECT *, year(ymd), symbol -FROM stocks WHERE year(ymd) IN ('2001', '2002', '2003') -``` - -If we execute this insert command, there is an error showing that the number of partitions that we're trying to create is more than the number of partitions that is allowed per node, which is by default set to 100, which is very low. But we can configure the number using two following properties: - -``` -SET hive.exec.max.dynamic.partitions=1000; -SET hive.exec.max.dynamic.partitions.pernode=500; -``` - -The first property `ive.exec.max.dynamic.partitions` is about how many partitions in total are allowed to be created. The second property `hive.exec.max.dynamic.partitions.pernode` is about how many partitions are allowed per node. - -Even though the two properties are set, when executing this insert, Hive may give an error because our cluster is not a huge cluster and when Hive is trying to calculate the space required to create all the necessary partitions, it will not find enough space in the cluster. So let's restrict the number of partitions which we are going to create using another condition in the where clause: - -```sql -INSERT OVERWRITE TABLE stocks_dynamic_partition -PARTITION (exch_name='ABCSE', yr, sym) -SELECT *, year(ymd), symbol -FROM stocks WHERE year(ymd) IN ('2001', '2002', '2003') and symbol like 'B%'; -``` - -For symbol that starts with `B`, this insert command will create partitions for records from year 2001 to 2003 and also for any stock symbol that is beginning with `B`. When the execution is completed, dynamic partition insert created a lot of partitions. To be specific, it created about 362 partitions. - -Let's check how the directories are structured for each partition. Since we have more than one partition columns in the table, so we do HDFS listing on the table first. - -![](assets/partitions-on-apache-hive_check-dynamic-partitions.webp) - -Under this table, we see the high level partition which is exchange name equals `ABCSE`. We can go into that partition and see what is inside that partition. As expected as shown in the above screenshot, there are three partitions under the first partition `ABCSE`: one for year 2001, 2002 and 2003. If we go into 2003 directory, there are several directories or partitions, one for each symbol. The symbol directories will have the files for that corresponding symbol. - -```sql -SELECT * FROM stocks_dynamic_partition -WHERE yr=2003 and volume > 10000; - -SELECT * FROM stocks_dynamic_partition -WHERE yr=2003 and sym='GEL' and volume > 10000; -``` - -The above queries are valid. In the first one, the MapReduce job targets the 2003 partition and it will process all the records under the directory. The second select is even better since the MapReduce job will only execute on files under the directory `GEL`, which is under the directory 2003, so the execution will be faster than the first query since we are scanning only fewer files. - -```sql -SELECT * FROM stocks_dynamic_partition -WHERE volume > 10000; -``` - -However, if we try to execute the above query, we will get an error because when the property `hive.mapred.mode` is set to strict. Hive will not allow to execute a query on a partition table without specifying a partition column in the where clause because this query is considered risky. Since we did not include a partition column in the where class for filtering, Hive has to scan the entire table to fetch the result set. To make this query work again, we can set this property to `non-strict` but it is not recommended because that would lead to performance issues. So, instead, let's fix the query by adding the partition columns in the where condition like this: - -```sql -SELECT * FROM stocks_dynamic_partition -WHERE exch_name = 'ABCSE' and volume > 10000; -``` - -And this query with the where condition exchange name and volume will execute with no issues. In summary, partitions minimizes the execution time by helping MapReduce job to execute on targeted files or directories. We also consider few different ways to load partitions. We also understand how to load hundreds of partitions with just a simple insert select statement using dynamic partitions. Partitions are very powerful and when we design the tables with right partition columns, it will save a lot of execution time and our queries will be faster. So when designing a Hive table next time, let's always think of partitions. - -## References - -- https://stackoverflow.com/questions/21876837/not-able-to-apply-dynamic-partitioning-for-a-huge-data-set-in-hive diff --git a/topics/data/redis-leaderboard.md b/topics/data/redis-leaderboard.md deleted file mode 100644 index ce72cc3ef..000000000 --- a/topics/data/redis-leaderboard.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: Redis leaderboard -description: Learn how to build a fast user leaderboard system using Redis sorted sets and hashes to track scores and store user profiles like usernames and avatars. -date: 2023-08-08 -github_id: Tuanpm31 -icy: 10 -redirect: - - /kDTinQ ---- - -In this post, we're going to explore how to implement a user leaderboard system using Redis. The leaderboard will keep track of user scores and profile information such as username and avatar. - -## Introduction - -Redis is a powerful in-memory data structure store that is used as a database, cache, and message broker. We'll be leveraging two particular data types in Redis: **sorted sets** and **hashes**. - -- **Sorted sets**: In Redis, a sorted set is a data type that has a set of unique elements where each element is associated with a score. It provides us with an efficient way to maintain a list of elements based on their scores, which is perfect for a leaderboard system. - -- **Hashes**: Redis hashes are the perfect data structure to store object-like items. We'll use hashes to store user's information like username, avatar type, etc. - -## Let's get started - -First, ensure you have Redis installed and running on your system. - -### Storing and retrieving user scores - -We use the `ZADD` command to add users to the sorted set. The command takes the sorted set name, the score, and the user's ID. For example, to add a user with the ID 'user1' and a score of 150, we can use: - -```bash -ZADD leaderboard 150 user1 -``` - -To retrieve the scores in descending order (highest score first), we can use the `ZREVRANGE` command with the `WITHSCORES` option: - -``` -ZREVRANGE leaderboard 0 -1 WITHSCORES -``` - -### Conditionally updating scores - -In many cases, we want to update a user's score only if the new score is higher than the existing one. Redis allows us to do this easily with the `ZADD` command by adding the `XX GT` option: - -``` -ZADD leaderboard XX GT 200 user1 -``` - -This command will update 'user1's score to 200 only if 200 is greater than their current score. - -### Storing and retrieving user information - -We can use the `HSET` command to store user information. For example, to set the username and avatar type for 'user1': - -``` -HSET user:user1 username "John Doe" avatar "TypeA" -``` - -To retrieve this information, we can use the `HGETALL` command: - -``` -HGETALL user:user1 -``` - -## Tying it all together - -Before we look at the specific commands to update both the sorted set and hash when a user achieves a new score, let's visualize the whole flow with the diagram below: - -![](assets/redis-leaderboard_flow_diagram.webp) - -This diagram depicts the flow as follows: - -- The user achieves a new score and sends it to the application. -- The application updates the user's info in the Redis hash. -- The application also updates the score in the Redis Sorted Set (the leaderboard). -- The Redis hash and sorted set return the updated user info and leaderboard to the application. -- The application displays the updated leaderboard to the user. - -Now that we know how to store and retrieve user scores and profile information, we can tie it all together. - -When a user achieves a new score, we can update both the sorted set and the hash: - -``` -ZADD leaderboard XX GT 200 user1 -HSET user:user1 username "John Doe" avatar "TypeB" -``` - -To display the leaderboard, we first retrieve the user IDs and scores: - -``` -ZREVRANGE leaderboard 0 -1 WITHSCORES -``` - -For each user ID, we then retrieve the user information: - -``` -HGETALL user:user1 -``` - -## Conclusion - -That's it! We've created a simple but effective leaderboard system using Redis' sorted sets and hashes. Redis is a powerful tool for such use cases due to its speed and efficient data structures. - -## Reference - -- https://redis.io/docs/data-types/sorted-sets/ -- https://redis.io/docs/data-types/hashes/ diff --git a/topics/data/self-balanced-bsts-avl-trees.md b/topics/data/self-balanced-bsts-avl-trees.md deleted file mode 100644 index 920b25567..000000000 --- a/topics/data/self-balanced-bsts-avl-trees.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Self balanced bsts avl trees -description: Learn how trees, especially binary search trees (BST) and AVL trees, organize hierarchical data for fast search, insertion, and deletion with guaranteed balanced height and efficient operations. -date: 2023-02-13 -github_id: mirageruler -icy: 10 -redirect: - - /ndx4Lw ---- - -## What are trees? - -- A **tree** is a non-linear data structure used to represent the hierarchical relationship between a paretn node and a child node. Each node in the tree is connected to another node by directed edges. -- ![](assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-team.webp) -- ![](assets/self-balanced-bsts---avl-trees_a-tree-representation-of-a-file-system.webp) -- Why do we use trees? The main advantage of using a tree over linear data structures like arrays or linked lists is that we do not have to search an element in linear team. - -## Important terms related to trees - -- ![](assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-1.webp) - - **Node**: The fundamental element of a tree. Each node has arbitrary data and two pointers that may point to null or its children. - - **Edge**: Another fundamental part of a tree used to connect two nodes. - - **Root**: The only node without incoming edges. It is the top node of a tree. - - **Leaf**: A node that has no children. - - **Path**: An ordered list of nodes that are connected by edges. - - **Height of a node** : The number of edges(in this case levels, everytime we go down by an edge we are going down one level of a tree) on the longest downward path between the node itselft to a leaf or the total number of nodes in that path exclude the node we're calculating the height for. Height of a binary tree is height of the root node. One more view to look at height of a node is to focus only the sub-tree where the focused node is the root, so it's height is the maximum depths of nodes within that sub-tree -- ![](assets/self-balanced-bsts---avl-trees_key-terminologies-in-a-tree-2.webp) - - **Parent**: A node is said to be a parent node if it has outgoing edges to other nodes. - - **Child**: A node that has incoming edges from another node is said to be the child of that node. - - **Sibling**: Nodes that are children of the same parent are called sibling nodes. - - **Ancestor**: A node is said to be an ancestor node if it is reachable while moving from child to parent. - - **Level/Depth**: The depth/level of a node is the number of edges on the path from the root node to that node. The level of the leaf on the longest path from root is also the height of the tree. Take this in general we can say height of node X is maximum depth in subtree of node X. - -## Type of trees - -- Generally, trees can be divided into two categories based on the number of children a node can have. They are as follows: - - **Non-binary trees** - - A non-binary tree is a type of tree in which a node can have **more than two children**. - - Examples: **2-3 trees**, **2-3-4 trees**,B-trees, B+ trees**, and **B\*\*\* trees are all examples of non-binary trees. But today, we will not care about these guys. - - ![](assets/self-balanced-bsts---avl-trees_non-binary-tree-example.webp) - - **Binary trees** - - A binary tree is a type of tree in which each node has at **most two** children. - - Examples: BSTs, AVL trees, and red-black trees are all examples of binary trees. - - ![](assets/self-balanced-bsts---avl-trees_binary-tree-example.webp) - -## Introduction to binary tree - -- As we discussed earlier, _a binary tree is a type of tree in which each node has at most two children_, which means a node in the binary tree can have one, two, or no children. These children are referred to as the **left child** and the **right child**. To give it a more general view, I would say every node in the tree all has a parent node, except the root or we could argue that parent of the root is nil or a nil node. -- ![](assets/self-balanced-bsts---avl-trees_a-tree-node-example.webp) - We can code the above node as follows: - -```go -type Node[T any] struct { - // the contents of this node. - value T - // left and right nodes. - left, right *Node[T] - // parent *Node[T] we're not going to use this field today, I just place it here since it's helpful for other type of trees and/or different problems. - // height of this node - height int -} - -type Tree[T any] struct { - // root of the tree - root *Node[T] - - // compare function - compare compareFunc[T] -} - -// compareFunc provide clients the ability to help us indicate how to compare the given data structure since we're allowing it to be anything. -type compareFunc[T any] func(T, T) int -``` - -## Properties of binary tree - -- The following are properties of a binary tree which are already prooved by the inventors... any way for a more intuitive view, I will provide a "sketch proof" for each of the properties and they are called proof by induction.If you interested in it, basically we take the base case that can not be false and we assume our proposition(which is $p_n$) is true then we proove that every $p_n$ implies $p_{n+1}$ - 1. The maximum number of nodes on a level i of a binary tree can be $2^i$, where $i \geq 0$. Sketch proof: Take the base case, at level 0 we have only one node that is the root itself so $2^0 = 1$ still hold true for our proposition, at level 1 we might have at most two children for the root so $2^1 = 2$ still hold true for our proposition, at level 2 we might have at most four children that are two from the root's left node and two from the root's right node so $2^2 = 4$ still hold true for our proposition... so on and so forth. - 2. The maximum number of nodes in a binary tree of depth `k` is $2^{k+1}-1$, where $k \geq 0$. Sketch proof: Take the base case, at level 0 we have only one node that is the root itself so $2^{0+1}-1 = 1$ still hold true for our proposition, at level 1 we might have at most 3 nodes that are the two new children from our original root so $2^{1+1} -1 = 3$ still hold true for our proposition, at level 2 we might have at most 7 nodes that are the two new children from the root's left child and the root's right child so $2^{2+1} -1 = 7$ still hold true for our proposition... so on and so forth. - 3. There is exactly one path from the root to any nodes in a tree. Because a each node is like a decision, imagine we stand before a decisive situation where there is only two options to go, we choose one then it will ask us to choose between the two new options, so on and so forth... so it's impossible for us to encounter a desired situation as long as we make at least one wrong decision in that path as that wrong decision will definitely leads us to a totally different branch. - 4. A tree with $n$ nodes has exactly $n−1$ edges connecting these nodes. Sketch proof: Take the base case, with one node that is the root itself and zero edge so $1-1=0$ still hold true for our proposion, with two nodes that are the root and either the left or the right child so we have one edge check $2-1 = 1$ still hold true for our proposition, with three nodes that are the root and its children so we have two edges to connect them all together so $3-1 = 2$ still hold true for our proposition... so on and so forth. - 5. The height of a complete/full (!= perfect) binary tree of $N$ nodes is $O(lgn)$. Sketch proof: Let's $n$ be the number of nodes in a complete binary tree and let $l_k$ denote the number of nodes on level $k$, where the levels are numbered 1,2,3, ..., h. The last level, $h$ represents the height of the tree. Note that: $l_k=2l_{k-1}$, i.e. each level has exactly twice as many nodes as the previous level (since each internal node has exactly two children). $l_1=1$ i.e. on the "first level" we have only node (the root node). Let's expand several case to find out the recurrence/pattern, $l_2=2, l_3=4,l_4=8,l_5=16,l_6=32...$ so the recurrence solves to $l_k=2^{k-1}$. Note also that the leaves are at the last level $l_h$, where h is the height of the tree, so from the previous bullets we know that the last level has $l_h=2^{h-1}$ nodes. The total number of nodes, $n$, in the tree is equal to the sum of the nodes on all the levels: $1+2^1+2^2+ ... + 2^{h-1}=n$, again let's expand several case to find out the recurrence on the left hand-side of the equation. We try with $$h=0, \quad \therefore 2^0 =1$$$$h=1, \quad \therefore 2^0+2^1=3$$$$h=2, \quad \therefore 2^0+2^1+2^2=7$$$$h=3, \quad \therefore 2^0+2^1+2^2+2^3=15$$so the recurrence is $2^h-1$. Therefore: $$2^h-1=n$$ $$\therefore 2^h=n+1$$ $$\therefore lg2^h=lg(n+1)$$ $$\therefore hlg2= lg(n+1)$$ $$\therefore h=lg(n+1)$$ Ok, so now we know the height of the tree we can do one more thing that is to compute the number of leaves $l_h$ in the tree, we observed earlier that $l_h=l^{h-1}$ so we can substitute the value of h in this expressions $2^{h-1}=2^h/2^1=2^{lg(n+1)}/2=(n+1)/2$. In conclusion, for a complete binary tree $h$ is $O(lgn)$ and the number of leaves $l_h$ is roughly half of the nodes $(n+1)/2$. In effect, perfect binary trees have the same properties as they obviously just have more nodes to complete the last level compare to the complete binary trees which have their last level not completely be filled. - - Notes: For convenient I will use the notation $lg$ as lograrithm of base 2 ($log2(n)$)since $lg$ also has two letters. - -## Types of binary tree - -- There are many types of binary trees. Let’s discuss them one by one below. - - **Complete binary tree** - - This is a very important invariant of binary tree that we need to keep in mind. In a complete binary tree, in every level except possibly the last, is completely filled. All nodes on the left are filled first and the nodes on the right filled after, so all nodes in the last level are as far left as possible. So all internal nodes have exactly two children and all leaves are at the same level. A binary min/max heap is a great example of a complete binary tree. - - ![](assets/self-balanced-bsts---avl-trees_a-complete-binary-tree-example.webp) - - **Full/Strict binary tree** - - The full binary tree is a binary tree in which each node has exactly zero or two children. - - ![](assets/self-balanced-bsts---avl-trees_a-full-strict-binary-tree-example.webp) - - **Perfect binary tree** - - The perfect binary tree is a type of full binary tree in which each non-leaf node has exactly two child nodes. All leaf nodes have identical path lengths and all possible node slots are occupied. - - ![](assets/self-balanced-bsts---avl-trees_a-perfect-binary-tree-example.webp) - - **Right-skewed binary tree** - - A binary tree in which either each node has a right child or no child (leaf) is called a **right-skewed binary tree**. - - ![](assets/self-balanced-bsts---avl-trees_a-right-skewed-binary-tree-example.webp) - - **Left-skewed binary tree** - - A binary tree in which either each node has a left child or no child (leaf) is called a **left-skewed binary tree**. - - ![](assets/self-balanced-bsts---avl-trees_a-left-skewed-binary-tree-example.webp) - - **Height-balanced binary tree** - - A **height-balanced binary tree** is a binary tree such that the left and right subtrees for any given node differ in height by a maximum one. AVL trees and red-black trees are examples of height-balanced trees. - ``` - Note: Each complete binary tree is a height-blanaced binary tree - ``` - - ![](assets/self-balanced-bsts---avl-trees_a-height-balanced-binary-tree-example.webp) - - - -## Binary search tree (BST) - -- A BST or a binary search tree is a binary tree in which nodes are ordered in the following way: - ![](assets/self-balanced-bsts---avl-trees_a-binary-search-tree-example.webp) - Below are the invariants of a BST (an arbitrary tree need to satisfy these invariants/constraints in order to be considered as a BST): - For all nodes x, if y is in the left sub-tree of x, we have key(y) <= key(x) - For all nodes x, if y is in the right sub-tree of x, we have key(y) >= key(x) - No duplicate key is allowed in the tree. - - Because a binary search tree is also a binary tree, all the algorithms of a binary tree are applicable to a binary search tree. - - `Note: There can be two separate key and value fields in the tree node(We can order the nodes by their key, and for any given key there's an associated arbitrary value). However, for simplicity of this article, we’ll consider the value as the key.` - - BST give us a several useful operations: - - Insertion in $O(h)$ - - Deletion in $O(h)$ - - Queries for exact key or the predecessor or successor if that key is not exist in $O(h)$. - - We can easily return a sorted linear data structure by performing an in-order traversal on the BST which is only $O(n)$. - -## How a BST can be helpful? - -- To talk about the reason behind existence of BST, I'd like to take a toy problem that you can imagine exists in all sorts of scheduling problems, it's a part of a runway reservation system: - We'll assume an airport with a single runway, and we can imagine this runway is pretty busy. There's obviously safety issues associated with landing planes, and planes taking off. And so there are constraints associated with the system, that have to be obeyed. And we have to build these constraints in, and the checks for these constraints into our data structure. - What we'd like to do is reserve requests for landings, and each of them are going to specify landing time called $t$. So, in particular we're going to add $t$ to the set $R$ of landing times if no other landings are scheduled within $k$ minutes (k can either be constant or variable that will be changed depends on other business logic, but for simplicity we'll assume k is constant). So we need to be able to perform an insert operation to the data structure. - We have the current notion of time, everytime we have a plane that's already landed, we'd like to remove that landing from the set $R$. So every once in a while, as time increments, we're going to be checking the data structure maybe 30 seconds or 1 minute, it doesn't matter, the fact that matters here is we must be able to perform a delete operation on the data structure. - So we have set $R$ described as above, we don't quite know how to implement it yet. Assume, we're dealing with large inputs, we'd like to do all of these operations in $O(lgn)$ time where $n$ is the size of set $R$. - ![](assets/self-balanced-bsts---avl-trees_runway-reservation-problem-example.webp) - - Let's list out so called, basic data structures, that obviously exists before BST like: unsorted array, sorted array, linked list (singly/doubly/circular is not important in this particular problem), hash table, heap. We're going to shoot them down with respsect to not being able to make the efficiency requirement of $O(lgn)$ time for all attached operations. - Unsorted list/array: Almost everything we want to do on this data structure is linear. Although for adding it only take constant time but to do the $k$ minutes check it's $O(n)$ time since the data have no order so we have to check with every other elements. Also for deletion, we have to traversed the whole data structure in the worst case (at the end of array) in order to find the element that we're going to delete. So basically, it's terrible. - Sorted list/array: I bet binary search is the first thing comes into your mind with this data structure. For the array implementation, yes we can do that to find the desired position to insert in $O(lgn)$ time and do the $k$ minutes check in $O(1)$ time but still take $O(n)$ time to do the insertion because possibily we have to shift every index after that, symmetrically the same is true for deletion. So it's almost what we want, if we can somehow skip the "shift index" work if we can manage our data in a more strictful structure. Actually in the list implementation, we don't have to do the "shift index" work but we can not do the "go to the middle point" operation since there's no indexes and we're only maintaing references a.k.a pointers. Give yourself a view of combining the key properties of a sorted array and linked list, that's the good things about the BST when it enables binary search as well as remove the $O(n)$ time "shifting" index work after the data structure is modified. - Heaps: Because of the logic of heap insertion and deletion methods, and the invariant of max/min heap (all children node must be larger/less than their direct parent) is fairly simple therefore fairly weak, each time we go left or right on the traversal we're not effectively cut off the all the unnecessary nodes, so to do the $k$ minutes check it's still take $O(n)$ time - that is we have to traverse every elements one by one in the heap. So it fails our requirement. - Hash table: Or dictionaries, hash map, whatever you wanna call it, supports $O(1)$ insertion, deletion, search exact all by a given key. But keys are not placed in order, actually they're in a randomized order so again if we want to do the $k$ minutes check it still $O(n)$ time. The good old hash table fails our requirement too. - - The beauty of a BST is that we can augment the tree to do more work as necessary without changing the efficiency of the attached methods. In short, we can easily do the $k$ minutes check while finding the place to insert. - - This should give us a sense of the richness of the BST structure, we can have nodes to store more data in them than just these common pointers. Having this notion of augmentation on the data structure is very good, because of design admendments so specifications never stay the same, like we're working for someone, and they never really tell us what they want, they might but they will change their mind later on, so in that case we're going to change our mind too and do the augmentation. I'll take an extra example requirements added upon our "runway reservation system" problem above, we need to compute $rank(t)$ that is how many planes are scheduled to land at times less than or equal to $t$, perfectly reasonable additional requirement and it wasn't part of the original spec. - - With the notion of the aumented BST, we can refer to the normal BST as the vanilla BST such that each node encapsulates common basic fields like left,right pointer, key... while the aumented BSTs are usually bring together other fields or callbacks needed to make the algorithms work. - - If you're interested in the implementations of BST you can search your own in the internet, today we're going to focus only on a method to balance BST. - -## BST needs to be balanced! - -- Almost operations for a vanilla BST is O(h) where h is the height of the BST but what we want is $O(lgn)$, so if $h = lgn$ where n is the total number of nodes in the BST then we will have $O(lgn)$ for operations attached to the BST which will truly satisfied our requirement for the "runway reservation system" problem. We have a concept of the balanceness of a tree, to say that a tree is balanced that is it's height must be $O(lgn)$, because almost . Because of the invariant of the BST that is not related to the height of the tree so that BST can become totally skewed to the right/left which is essentially a linked list and we know what's the problem with a linked list. Now this becomes our new problem as obstacles that we have to solve before we can have a data structure that efficiently acchieve $O(lgn)$ time in all attached operations in most of the time. - -## The height of a node - -- The new question is, what do we need to do in order to guarantee the height of a BST to be $lgn$. And before getting to methods that help us to actually do that, I think it's useful to discuss about height of the BST or in general height of a node within a BST because height of node root is also height of the BST. -- I will give out a hint that AVL require each node to store an additional integer field call `height` to store the height of each node, because the algorithm need to know that information instaneously in order to effecienly balance the tree (Simply we can't afford to go down the tree or sub-trees if you will to compute the height every time we need it). For a more intuitive view, let's look at the following picture: -- ![](assets/self-balanced-bsts---avl-trees_height-of-a-node-in-a-bst.webp) -- We need to update the height of each node in the AVL tree every once in awhile whenever insertions or deletions happen on the tree a.k.a the tree structure is modified. - -## How to balance BST? - -- BSTs are augmented and added invariants in many different ways in order to achieve balanceness in their height and they have a common name "Self-balance binary search tree", the most famous are AVL trees which we're going to talk about today, Red-Black trees, Splay trees, treaps. Each type of self-balance BST either has their own idea (Red-Black trees have the idea of coloring nodes with red or black) or common idea (the node rotation is commonly used in variants of self balance BST) to achieve balanceness in height. - -## AVL tree - -- A little bit information about the origin of this data structure: - - Named after two Soviet inventors **A**delson-**V**elsky a computer scientist and **L**andis a mathematician. - - It was the first self-balance BST ever to be invented and was the original way people found to keep trees balanced back in the '60s so they're kind of the simplest among others. - - AVL trees are often compared with Red-Black trees because both support the same set of operations and take $O(lgn)$ time for the basic operations. For lookup-intensive applications, AVL trees are faster than red–black trees because they are more strictly balanced. -- AVL tree has a little rule for the height of their nodes. That is if a node is missing its left or right child or both and the missing children is considered as an imaginary node and these nodes have height $h=-1$. In order for the formula to calculate the height of a binary node in a BST work with that base case. - -## Rep invariant of an AVL - -- I will go with the definition of the authors of the algorithm first. -- `First the tree itselft must be a BST, then the height of the left sub-tree for every nodes within the tree must not 1 apart from the height of the right sub-tree` -- Notice if we slightly change the above constraint to "must not 0 apart ..." which means the left sub-tree and the right sub-tree have the same height in effect essentially describe a perfect BST but this constraint is really hard to conform since there are only a certain number of nodes $n$ with a unique structure for each $n$ to be able to represent a perfect BST (level 1 require 1 node, level 2 require 3 nodes, level 3 requires 7 nodes, level 4 requires 15 nodes, level 5 requires 31 nodes... and with that number of nodes they have to presented in a perfect way so it's really hard to keep that, for example if we have 8 nodes then it's impossible to fix the invariant) -- The above definition by words can be expressed as mathematical as follows: $\forall n\in AVL, |h(n.left) - h(n.right)| \leq 1$ -- This constraint will help ensures the height of the AVL tree in the worst case of its definition stays $O(lgn)$ or in other words the highest height of an AVL tree is bounded by $O(lgn)$. In my humble opinion, It's important to understand how this is achieved by the invariant of the AVL tree in order to fully understand the idea of AVL trees, so I'll try my best to proove this is true: - - The first claim is that `AVL trees are balanced`, balanced means height is always $O(lgn)$, so we're just going to assume for now that we can somehow achieve this property and we want to prove that it implies that the height is at most some constant times $O(lgn)$ while we know it's at least $O(lgn)$ - - Let's think about the worst case for height of an AVL tree, say if we have $n$ nodes how could we make the AVL tree as high as possible? Or conversely, if we have a particular height, how could we make it have as few nodes as possible? That'd be like the sparsest or the least balanced situation for AVL trees. - - To achieve the worst case, we can do this: For every node, let's the right side have a height of $1$ larger than the left side or symmetrically we can do the same on the conversely but I'll only take one example here. - - We're going to define $n_{h}$ is the minimum number of nodes that's possible in an AVL tree of height $h$. This is sort of the inverse that we care about, but if we can solve the inverse, we can solve the thing. What we really care about is, for $n$ nodes, how large can the height be, we wanna prove that is bounded to $O(lgn)$. But it'll be a lot easier to think about the reverse, which is, if we fix the height to be $h$ then what's the fewest nodes that we can pack in? Because for a worst case of a BST a right/left degenerated BST, we have a height of $n$ so we only need to put $n$ nodes and we see that woulld be really bad. What we prefer is a situation where with heigh $h$, we have to put in $2^h$ nodes and that would be perfect balance, so when we take the inverse from that exponential equation we get a logarithm. - - Imagine a case where we want to build an AVL tree with a fixed height, and we want to have as few nodes as possible. If we have a tree with a big height and very few nodes, $h$ is going to be bad when we write it as a function of $n$ so those trees are unbalanced - big height, small number of nodes. So we will try to build up the next presentation of the AVL tree with minimum number of nodes for each level, let's look at the following picture: - - ![](assets/self-balanced-bsts---avl-trees_minimum-number-of-nodes-for-an-avl-tree-of-height-h.webp) - - Expanding the example to level 4,5,6,7 then you will regconize the pattern/recurrence. If we want to build an AVL tree with as few nodes as possible and height $h$, we start with the root and then at the right we build an AVL tree of height $h-1$ and at the left, we build an AVL tree of height $h-2$, because of the fact that we were also try to build left AVL sub-tree and right AVL sub-tree with minimum number of nodes then it turns out that the whole tree must has minimum number of nodes. Look at the following picture: - - ![](assets/self-balanced-bsts---avl-trees_the-best-way-to-build-a-tall-avl-tree-with-as-few-nodes-as-possible.webp) - - Suppose we want to write the number of nodes as a function of height, we have $n_h=n_{h-2}+n_{h-1}+1$ as the recurrence with the base case $n_{O(1)}=O(1)$. Now we need to solve it, what we would like is for it to be exponential, because that means there's a lot of nodes in a height $h$ AVL tree. Look at the recurrence, doesn't it look like something very familiar to our developers when we first come to some of the first problem when we learn how to write code? Yes, the good old Fibonacci, it's almost Fibonacci except we have our additional $+1$ in the recurrence. Well, that's actually good, because in particular, $n_h$ is bigger than Fibonacci. So we have $n_h>f_h$, if we add $1$ at every single level then certianly we get something bigger than the base Fibonacci sequence. Now, hopefully you know Fibonacci is exponential. Let's bring something that is already prooved in order to help us reduce the work here, we know that $f_h=\frac{\varphi^h}{\sqrt{5}}$ (refs:https://en.wikipedia.org/wiki/Fibonacci_number) the above method require rounding the result into the nearsest integer in order to give the exact result of the Fibonacci sequence, crazzy stuffs and obviously we don't need to know why that's true, just take it as fact. And conveniently $\varphi>1$, also we don't need to remember what $\varphi$ is($\approx1.618$), except it is bigger than 1. So we have an exponential bound, this is good news. What we really wanna know is how $h$ relates to $n$, which is just inverting the formula. So we have, on the other hand, $\frac{\varphi^h}{\sqrt{5}}2n_{h-2}+1$ because if we have a larger height we're going to have more nodes. We can even get rid of the $1$ because that only makes things bigger, so we have $n_h>2n_{h-2}$ . Now from this version of the recurrence, let's use inductive reasoning to solve it. -- All right, so far so good, now the next big question is how the heck can we maintain this AVL properties for our good old BST? - -## How to maintain the AVL invariant when the BST tree is modified - -- **Insertion**: - - Step 1: Do simple BST insertion, and this one will not preserve the AVL property. - - Step 2: Calculate the balance factor of the sub-tree related to the newly inserted node to check if it violates the AVL properties on that sub tree then we're going to fix that recursively when we going up from the first bottom of the recursive function. -- **Deletion**: - - Step 1: Do simple BST deletion, and this one will not preserve the AVL property. - - Step 2: Calculate the balance factor of the related sub-tree after the node is deleted to check if it violates the AVL properties on that sub tree then we're going to fix that recursively when we going up from the first bottom of the recursive function. -- Ok, sound like we're gonna have ways or precisely tools to fix the AVL properties, and that tool is **rotations**, super cool tools. -- ![](assets/self-balanced-bsts---avl-trees_rotations-for-binary-tree-nodes.webp) -- For left-rotate, whatever the parent of $X$ was becomes the parent of $Y$ and vice versa, in fact. The parent of $Y$ was $X$, and now the parent of $X$ is $Y$. The parent of $A$ is still $X$, the parent of $B$ changes, it used to be $Y$ now it's $X$. The parent of $C$ was $Y$, it's still $Y$. We call it left-rotate because the root moves to the left. -- For right-rotate, it a reverse operation with the left-rotate that let us manipulate the tree assymetrically. -- So in a constant number of pointer changes, we can perform what so called node rotation and more importantly it satisfies the BST order property, if we do an in-order traversal of this we will get $A,X,B,Y,C$ at the first state, not just a coincident the same is true for the in-order traversal of the second state. $B$ was some number of nodes between $X$ and $Y$, and it still some number of nodes between $X$ and $Y$, and so on you can expand the example tree and check more on yourself. -- In my humble opinion, it's like the only thing you need to know in BST along with how to do the search. -- Ok that's rotations by pictures, to talk about them in detail, we have to look at their specific cases: - - Case 1: I call this case "right child skewed to the right" - - ![](assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-right.webp) - - We rotate-left self (parent of the right child that is skewed) to balance the sub-tree - - Case 2: I call this case "left child skewed to the left" - - ![](assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-left.webp) - - We rotate-right self (parent of the left child that is skewed) to balance the sub-tree - - Case 3: I call this case "right child skewed to the left" - - ![](assets/self-balanced-bsts---avl-trees_right-child-skewed-to-the-left.webp) - - We rotate-right the right child that is skewed to achieve case 1, then we perform solution for case 1. - - Case 4: I call this case "left child skewed to the right" - - ![](assets/self-balanced-bsts---avl-trees_left-child-skewed-to-the-right.webp) - - We rotate-left the left child that is skewed to achieve case 2, then we perform solution for case 2. -- The common pitfall here is to think that we only need to fix that one violated local sub-tree that related to the insertion/deletion, where we only fix the lowest violation of the AVL property there maybe violations higher up. We need to fix the violation all the way up to the root from the insertion/deletion point. -- Suppose $X$ is the lowest node violating the AVL properties, the way we find this node is we start at the node that we changed (insert/delete), we update the height based on the heights of its children and check if it's ok as we go up, and we keep walking up until we see, oh, the left is $+2$ larger than the right or vice versa, then we fix it. -- Phew, it's a long article though. Hope you get some sense of the idea of AVL trees. If you looking for an example implementation of it, I've coded a version of an AVL tree for storing integers in Go you can take it as reference: https://github.com/mirageruler/Data-Structures-and-Algorithms-Implementations/blob/main/data_structures/tree/avl.go. In the near future, I'm looking forward to augment it to support generic types and see if I can leverage concurrency to speed up the algorithms which mean to deal with all sorts of the concurrency related problems. - -## References: - -- https://en.wikipedia.org/wiki/AVL_tree -- https://www.youtube.com/watch?v=9Jry5-82I68&list=PLUl4u3cNGP61Oq3tWYp6V_F-5jb5L2iHb&index=5 -- https://www.youtube.com/watch?v=FNeL18KsWPc&list=PLUl4u3cNGP61Oq3tWYp6V_F-5jb5L2iHb&index=6 -- https://www.youtube.com/watch?v=r5pXu1PAUkI&list=PL-K_ib5mxHXnukhVpx_wMun21O2GQUEE0&index=5 -- https://www.youtube.com/watch?v=IWzYoXKaRIc&list=PL-K_ib5mxHXnukhVpx_wMun21O2GQUEE0&index=6 diff --git a/topics/data/sql-practices-orm-vs-plain-sql.md b/topics/data/sql-practices-orm-vs-plain-sql.md deleted file mode 100644 index cc0b39420..000000000 --- a/topics/data/sql-practices-orm-vs-plain-sql.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: Sql practices orm vs plain sql -description: Learn how to choose between raw SQL, ORM, and SQL generators, optimize database performance with smart indexing, avoid rounding errors, and write efficient queries for better security and speed. -date: 2021-07-07 -tags: - - database - - sql -redirect: - - /eaanGw ---- - -## Definition - -- Raw: make all query by self from scratch ORM: Object relational mapping: programing technique for converting data between incompatible type systems using OOP -- SQL Generator: utility sql tool for generate Entity model, query. - -## Factors that affects the technique decision - -- Productivity: spend time for development and maintain. Work with many different DB system, can replace in future -- Security: SQL injection -- Performance - -### Practice - -Use ORM for almost case, with DEEP understanding the library. Raw is supported in ORM Use Raw and SQL Generator for some special case - -## Indexing - -The best technique for improve performance in database is to use indexes well. A index is a data structure that the database uses to correlate value to the rows where these values occur in a given column. An index provides an easy way for the database to find values more quickly than the brute-force method of searching the whole table from top to bottom. Index can also help an Update or Delete statement by finding the rows quickly (primary key) - -### Misunderstanding when using indexes - -- Index Aren't Standard: ANSI SQL standard says nothing about indexes. -- Defining no indexes or not enough indexes -- Defining too many indexes or indexes that don't help -- Running queries that no index can help - -Using INSERT, UPDATE, DELETE, the database has to update the index data structures for that table to be consistent so that our subsequent searches use these indexes to find the right set of rows reliably. - -There's no benefit to creating indexes that we don't use - -- Index on primary key column -- Index for a long string type -- Not going to search for specific values(datetime) -- Make compound indexes that are redundant or seldom used (can use join, search, sorting order criteria) - -### Practice - -Indexing base on the requirement or what queries are important to optimize - -Measure the application code to find out the bottle-neck. Don't make informed decisions without information. - -- Tool: pgfouine -- Using `Explain` to make a report of Query Analysis - Query Execution Plan(QEP). - -### Make index with condition - -CREATE UNIQUE INDEX line_items_prod_var_null_idx - -ON line_items (product_id) - -WHERE variant_id IS NULL - -### Rebuild the index data - -Over time, as we update and delete rows, the indexes may become fragmented overtime - SQL system tool - -### Rounding errors - -Fractional Number(float, decimal) is common type in db: money, measurement: length, weight, capacity, temperature, time. infinite precision VS finite precision: 1/3 vs 0.33 - -IEEE 754 represents floating-point numbers in a base-2 format. The Float data can't represent exactly in binary. The reason is we try to convert base-10 to base-2 - -See more reference at [Oracle’s document.](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html) - -### Practice - -Use Numeric or Decimal in SQL for fixed-precision fractional numbers - -### Query practice - -- Use alias make the more readable query -- Prefer `=` than `like` than `like 'prefix%'` than `like '%content%'` -- Using `WITH` for complex query -- Avoid `DISTINCT`, should verify the reason of duplicated record -- Avoid `Select *`, spell out all columns you need(Select, Insert) - -Get row with Greatest value per group. Follow the Single-Value rule to build the query. The rows in each group are those rows with the same value in the column or columns you name after GROUP BY. Every column in the select-list of a query must have a single value row per row group. - -## Search a simple text - -- Optimize search text in db -- Poor performance solution: wildcard (%) that matches zero or more characters. - -### Practice - -- Full-text search: support from db -- Third-party search engines - right tool for the right job -- Implement from scratch: `XXX`, `Keywords`, `XXXKeywords`. Create a procedure for searching. Make a trigger for update `Keywords` and `XXXKeywords` when Update or Insert data to `XXX` table - -### Reference - -- [pragprog.com/titles/bksqla/sql-antipatterns](https://pragprog.com/titles/bksqla/sql-antipatterns/) -- [www.enterprisedb.com/blog/postgresql-query-optimization-performance-tuning-with-explain-analyze](https://www.enterprisedb.com/blog/postgresql-query-optimization-performance-tuning-with-explain-analyze) -- [technical-recipes.com/2011/ieee-754-floating-point-to-binary-conversion](http://technical-recipes.com/2011/ieee-754-floating-point-to-binary-conversion) diff --git a/topics/data/sql-sargable-queries-and-their-impact-on-database-performance.md b/topics/data/sql-sargable-queries-and-their-impact-on-database-performance.md deleted file mode 100644 index 9d0c8a561..000000000 --- a/topics/data/sql-sargable-queries-and-their-impact-on-database-performance.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: SQL saragable queries and their impact on database performance -description: Structured Query Language (SQL) is widely used for managing data in relational database management systems (RDBMS). In the context of SQL, the optimization of database queries forms the foundation for efficient data retrieval, providing quicker response times and increasing overall database performance. One key strategy to optimize SQL queries involves the concept of "Sargable" queries... -date: 2023-09-14 -authors: - - monotykamary -github_id: monotykamary -tags: - - database - - performance - - sql - - sargable-queries -redirect: - - /ll5yWg ---- - -Structured Query Language (SQL) is widely used for managing data in relational database management systems (RDBMS). In the context of SQL, the optimization of database queries forms the foundation for efficient data retrieval, providing quicker response times and increasing overall database performance. One key strategy to optimize SQL queries involves the concept of "Sargable" queries. - -"Sargable" is derived from "Search ARGument ABLE," signifying that a query can successfully utilize indexes for efficient execution. It is a term that was initially introduced in a 1979 research paper titled "Access Path Selection in a Relational Database Management System" authored by P. Griffiths Selinger et al. (DBA.StackExchange.com). The fundamental idea behind sargable queries is geared towards forming SQL statements that allow the query execution engine to make the best use of indexes whenever available. - -## Sargable vs. non-sargable queries - -A central feature differentiating sargable from non-sargable queries is the manner in which operations are executed on indexed columns. Non-sargable queries include function calls or operations that use an indexed field in the WHERE clause, a feature that hinders the usage of indexes. - -| Non-sargable Query | Sargable Query | -| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | -| `SELECT * FROM myTable WHERE SUBSTRING(myColumn, 1, 3) = 'ABC';` | `SELECT * FROM myTable WHERE myColumn LIKE 'ABC%';` | -| `SELECT * FROM myTable WHERE DATEDIFF(day, myDate, GETDATE()) = 7;` | `SELECT * FROM myTable WHERE myDate >= DATEADD(day, -7, GETDATE()) AND myDate < GETDATE();` | -| `SELECT * FROM myTable WHERE ISNULL(myColumn, 'N/A') = 'N/A';` | `SELECT * FROM myTable WHERE myColumn IS NULL;` | - -### Wildcards - -Attempting to use wildcards at the beginning of a string in a LIKE clause also tends to create non-sargable conditions. For instance, the condition `WHERE name LIKE '%prefix%'` usually results in a table or index scan, which is significantly slower than an index seek. - -Let's consider a simple example. Suppose we have a table called "Customers" with columns "CustomerID", "Name", and "Address". We want to find all customers who live in a particular city. A non-sargable query would be: - -```sql -SELECT * FROM Customers WHERE Address LIKE '%New York%'; -``` - -This query is not sargable because the LIKE operator does not allow the query engine to use an index on the Address column. However, we can modify the query to make it sargable: - -```sql -SELECT * FROM Customers WHERE Address = 'New York'; -``` - -Now, the query engine can use an index on the Address column to optimize the search process. - -### Handling NULL values - -Another important aspect of sargable queries is handling NULL values properly. The returned dataset will be empty if we do not handle NULL values correctly. For example, consider the following query: - -```sql -SELECT * FROM Customers WHERE Name IS NOT NULL; -``` - -This query is not sargable because the `IS NOT NULL` predicate does not allow the query engine to use an index. To make this query sargable, we can modify it to: - -```sql -SELECT * FROM Customers WHERE Name IS NOT NULL AND Address IS NOT NULL; -``` - -Now, the query engine can use an index on the Name and Address columns to optimize the search process. - -### Calling with functions - -A non-sargable query may look like `WHERE YEAR(dateColumn) = 2022`. The problem with such a query is that it requires evaluation of the function `YEAR(dateColumn)` for each row in the table, thus preventing us to use any pre-existing index on `dateColumn`, an operation leading to inefficient table scans. - -Sargable queries aim at performing operations responsibly by avoiding function calls on indexed columns whenever possible. For instance, reversing the non-sargable condition from - -```sql -SELECT * -FROM myTable -WHERE YEAR(dateColumn) = 2023; -``` - -to - -```sql -SELECT * -FROM myTable -WHERE dateColumn >= '2023-01-01' AND dateColumn < '2024-01-01'; -``` - -allows the query optimizer to use our indexes and not run the function on every row. - -## Impact on performance - -The distinction between sargable and non-sargable queries lies primarily in how efficiently they facilitate usage of indexes. Sargable queries allow the database engine to perform index seeks, a process whereby only the matching data in index pages are read, hence reducing the consumption of [[SQL and how it relates to Disk Reads and Writes|input/output (IO) resources and time]]. - -Several advantages come from employing sargable queries. The main benefit lies in leveraging indexes, thereby improving search speed. Depending on the data type and column values, diverse index types such as clustered indexes, non-clustered indexes, and columnstore indexes can be used. - -## Conclusion - -In modern databases, where massive data sizes are common, improving query performance can significantly impact the overall system speed and efficiency. Sargable queries provide a crucial optimization strategy that should be taken into account during SQL programming and query design. - -By understanding how sargable queries can take advantage and efficiently use indexes, database administrators, and developers can significantly boost search speeds and performance. Furthermore, ways to convert non-sargable queries into sargable queries by avoiding function calls on indexed columns and leveraging efficient use of the LIKE operator and wildcards should be studied for regular practice. - -## References - -- https://stackoverflow.com/questions/799584/what-makes-a-sql-statement-sargable -- https://www.sqlshack.com/how-to-use-sargable-expressions-in-t-sql-queries-performance-advantages-and-examples/ -- https://www.tech-recipes.com/uncategorized/sargable-queries-in-sql-server-with-examples/ -- https://www.mssqltips.com/sqlservertip/6795/improve-sql-server-query-performance-searchable-arguments/ -- https://dba.stackexchange.com/questions/162263/what-does-the-word-sargable-really-mean diff --git a/topics/data/statistics-in-data-analysis.md b/topics/data/statistics-in-data-analysis.md deleted file mode 100644 index 432ce2c5c..000000000 --- a/topics/data/statistics-in-data-analysis.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Statistics in data analysis -description: Learn how key statistical distributions like uniform, normal, and Pareto help analyze data accurately and make better business decisions with meaningful insights. -date: 2022-10-24 -github_id: Bachpx2911 -redirect: - - /KUatfg ---- - -## What is Statistics? - -### Statistics in data analysis - -#### Distribution - -Statistical methods are mainly useful to ensure that your data are interpreted correctly. And that apparent relationships are really significant or meaningful and it is not simply happen by chance. Actually, the statistical analysis helps to find meaning to the meaningless numbers. One of important concept of statistic is distribution. This article will give you some example for applying distribution in data analytic - -#### Uniform distribution - -A distribution in statistics is a function that shows the possible values for a variable and how often they occur. One of the basic distribution is uniform distribution, which measure for events having the same probability of happening. For instance, if your company create a marketing campaign for 100 first customers get the reward an Iphone Promax, the probability of each customer winning the price is equal 1 %. - -#### Normal distribution - -Other distribution is normal distribution, which is an arrangement of a data set in which most values cluster in the middle of the range and the rest taper off symmetrically toward either extreme. The example for normal distribution is that the height or weight of all people in nation. However, normal distribution can be a trap because some data is misunderstanding. For instance, when people analyze salary of industry or company, they often say that the average salary is around 10000 USD per year and increase fast by year. However, there are some people whose salary are higher than 1 000 000 USD per year and other employee with salary are around 1000 USD per year. The company may boast that it doubled the average salary of its employees, but actually increased the salary of the leader 10x but only increased the salary of the employees by 1% - -### Pareto distribution - -Another type of distribution applied in data analytic is Pareto distribution. It measures the case that a small value event occur frequently and high value event occur rarely. In marketing analyst, they can use this distribution to find customer segmentation, which lead them to focus on important customer. Other instance is in risk modelling. Risk modeler need to define and calculate probability, stress test of some low probability event that can cause serious effect on system - -## Conclusion - -From my point of view, mastering distribution will bring huge benefit in understanding data. Then, a good decision can be made for your business - -## Reference - -- https://thebusinessprofessor.com/en_US/research-analysis-decision-science/normal-distribution-definition -- https://www.sciencedirect.com/topics/computer-science/pareto-distribution diff --git a/topics/defi/AMM.md b/topics/defi/AMM.md deleted file mode 100644 index 17ba4df8a..000000000 --- a/topics/defi/AMM.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Amm -description: null -date: null -redirect: - - /8BpYYQ ---- - -### Automated market maker - -- Tools to create automated market -- Operate on Decentralized Exchange Platform [[¶ Decentralized Exchange]] -- Based on mathematics formulas (x\*y=z) to estimate/ evaluate a Token's value - -x\*y= z: A constant number that helps create a balance status for the system diff --git a/topics/defi/AtomicSwap.md b/topics/defi/AtomicSwap.md deleted file mode 100644 index 3ab74de05..000000000 --- a/topics/defi/AtomicSwap.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: AtomicSwap -description: null -date: null -redirect: - - /evDpCg ---- - -### What - -Atomic swap is the swap of 2 coins on a separate blockchain system, based on [[Smart Contracts]], allowing users to trade coins directly from the wallet. => Peer-to-peer transactions between different blockchains. - -### Why - -Cross-chain trading can eventually solve many of the problems that are part of most centralized exchanges **Greater vulnerability** keeping many valuable resources in one location makes them more vulnerable to hacking, and centralized exchanges are prime targets for digital heists. => lưu lượng tiền lớn -> dễ bị hack (trứng cùng một rổ) **Funds mismanagement and human error** maintain by human => cause error sometimes **Higher operational costs** **Inefficiency in regards to volume demands:** khối lượng giao dịch nhiều => thường bị nghẽn do ko giải quyết được hết **Regulation** legal - -Cross-chain trading can eventually solve many of the problems that are part of most centralized exchanges **Greater vulnerability** keeping many valuable resources in one location makes them more vulnerable to hacking, and centralized exchanges are prime targets for digital heists. => Big cash flow -> Big chance of getting hacked **Funds mismanagement and human error** maintain by human => cause error sometimes **Higher operational costs** **Inefficiency in regards to volume demands:** High transaction volume => Cause blocking in transaction **Regulation** legal - -### Advantage - -By removing the need for a centralized exchange or any other kind of mediator, cross-chain swaps can be executed by two (or more) parties without requiring them to trust each other - -- increased level of security because users don’t need to give their funds to a centralized exchange or third party -- lower operational costs as trading fees are either very low or absent -- happen quickly (altcoins can be swapped directly without making use of Bitcoin or Ethereum as an intermediary coin.) - -### Limit - -For instance, to perform an atomic swap, the two cryptocurrencies need to be based on blockchains that share the same hashing algorithm (e.g., SHA-256 for Bitcoin). Also, they also need to be compatible with HTLC and other programmable functionalities. bring up concerns about users privacy (?) -> on-chain swaps and transactions can be quickly tracked on a blockchain explorer, making it easy to link the addresses. - -### How it works - -(https://www.youtube.com/watch?v=WkXUz3UFn6Y) - -A swap his LTC to receive B's BTC A send a contract address, which requires a key to access. A send a hash of this key for B => B can't access the LTC because he only has the hash of key - -B uses the provided hash to create another contract address, in which he deposits BTC. To claim the BTC, A requires the same key. As soon as A claim the BTC, B is able to claim the LTC. - -If any party gives up or fails to do, the contract is canceled, the funds are automatically returned to their owners. - -**On chain** Happen on either of the currency's network. (In this case, it's Bitcoin and Litecoin) - -**Off-chain** Off-chain atomic swaps, on the other hand, take place on a [secondary layer]. This kind of atomic swaps is usually based on bidirectional payment channels, similar to the ones used in the [Lightning Network] - -**Hash Time Lock Contracts** Hash Lock: prevents money from being spent except for predefined data Time Lock: ensures the contract is only executed within a certain time frame => Create a set of rules to prevent partial swap. diff --git a/topics/defi/BIP39.md b/topics/defi/BIP39.md deleted file mode 100644 index 93a355702..000000000 --- a/topics/defi/BIP39.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: BIP39 -description: null -date: null -redirect: - - /7ltqDw ---- - -- BIP39 is the protocol to generate mnemonic sentence (e.g seed phrase in metamask) -- Under the hood, from that seed phrase will generate a key wallet, if you want to generate the following keys, you will be input from seed phrases + (key1+key2) (for example with metamask I can generate many key wallets, later on When you re-import the seed phrases on another device, those keys will also be re-generate from the seed phrases without having to remember them) -- Bip39's dictionary bank has 2048 words -- Entropy of 1 metamask with 12 phrases is 128 -> There are 2^128 possible phrases. If we brute force 1 seed / 1 microsecond it will also take 1.07 \* 10^23 years - ---- - -#### Reference - -- https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki -- https://www.reddit.com/r/ledgerwallet/comments/6cjvam/probability_of_seed_phrase_collisionbrute_forcing/ diff --git a/topics/defi/Cryptoeconomics.md b/topics/defi/Cryptoeconomics.md deleted file mode 100644 index a760b2f27..000000000 --- a/topics/defi/Cryptoeconomics.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Cryptoeconomics -description: null -date: null -redirect: - - /WNxdbA ---- - -### Cryptoeconomics - -Cryptoeconomics offers a way to coordinate the behavior of network members to solve the problems of coordinating members in a digital ecosystem using cryptography and economic leverage. - -Cryptoeconomics is not a subset of traditional economics but a blend of game theory, the field of mechanism design, mathematics, and methods of economics. The main goal is to understand how to fund, design, develop, and facilitate the operation of decentralized networks [[¶ DeFi (Decentralized Finance)]]. - -#### The cryptoeconomic circle - -The cryptoeconomics circle is an overarching model of cryptoeconomics. It was published by _Joel Monegro_ and illustrates abstract value flows through different membership classes in a peer-to-peer economy. Utility: Reference from [AXS](https://research.binance.com/en/projects/axie-infinity) - -It represents a 3-sided market: - -- Among miners (supply side) -- Users (demand side) -- Investors (funding side) - -Those teams exchange the value with each other using a cryptocurrency resource token - -Investors have two roles in this model: providing liquidity to miners to sell their tokens, and taking advantage of the network by backing tokens that cost more than mining costs. - -The model represents these two roles by dividing investors into two groups: traders (short-term investors) and owners (hodlers) (long-term investors). - -#### Funding - -When investing, we will think about **funding team** and **funding network**. The funding team provides the financial capital to build the service. Funding networks support growth by raising capital from the whole community. They are very different forms of investment, but both are essential to long-term network success. - -#### What problems can be solved? - -Before Bitcoin, it was considered impossible to create a peer-to-peer network, in which the consensus can be achieved without holes or errors. - -This problem is often referred to as the Byzantine General Problem. It is a logical dilemma, showing that in distributed systems it is very important for actors to reach agreement. The assumption of this problem is that, because some actors may be unreliable, agreement can never be reached and the network cannot function as intended. - -With the creation of Bitcoin, Satoshi Nakamoto brought economic leverage into the peer-to-peer network and solved this problem. - ---- - -#### Reference - -- https://www.placeholder.vc/blog/2019/1/5/the-cryptoeconomic-circle -- https://www.placeholder.vc/blog/2019/1/10/funding-cryptonetworks -- https://academy.binance.com/vi/articles/what-makes-a-blockchain-secure diff --git a/topics/defi/DefiApplication.md b/topics/defi/DefiApplication.md deleted file mode 100644 index 218ab2523..000000000 --- a/topics/defi/DefiApplication.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: DeFi application -description: null -date: null -redirect: - - /WWnOxw ---- - -### Lending - -DeFi allows any individual to take out or supply a loan without approval from a third party. - -- [AAVE](https://app.aave.com/markets) -- [FulCrum](https://fulcrum.trade/) -- [BlockFi] -- [Compound] -- [Nexo] -- [CoinList] -- [Curve] -- [dYdX] -- [yEarn] -- [Maker] -- [mStable] -- [Nuo Network] - -### Decentralized exchange - -Decentralized exchanges allow users to swap their assets without having to transfer custody of the underlying collateral - -- [0x Protocol] -- [1inch] -- [Bancor] -- [dYdX] -- [Kyber Network] -- [Oasis] -- [Open Sea] -- [Matcha] -- [Uniswap] - -### Derivatives - -In traditional finance, a derivative represents a contract where the value is derived from an agreement based on the performance of an underlying asset. - -- [Augur] -- [dY/dX] -- [Erasure] -- [Opium] -- [Opyn] -- [Synthetix] -- [UMA] - -### Wallet - -Wallets are a crucial gateway for interacting with DeFi products. - -- [Argent] -- [Authereum] -- [Coinbase] -- [Magic] -- [Metamask] -- [Portis] - -### Asset management - -Tools are in place to better track and manage assets - -- [Balancer] -- [Defi Saver] -- [InstaDapp] -- [Sablier] -- [Set Protocol] -- [Zerion] -- [Zapper] - -### Savings - -Offer unique and novel ways to earn a return by saving cryptocurrencies. This differs from lending as there is no borrower on the other side of the table. - -- [DAI Saving Rate] -- [Dharma] -- [Pool Together] diff --git a/topics/defi/Definition.md b/topics/defi/Definition.md deleted file mode 100644 index 53e34e1b8..000000000 --- a/topics/defi/Definition.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Definition -description: null -date: null -redirect: - - /XLTguA ---- - -### What - -DEX (Decentralized exchange) or decentralized exchange, in this type of exchange, all transactions take place automatically and directly between users (peer-to-peer network) according to an automated process that Don't depend on 3rd party or middleman to keep your funds. In other words, users have complete control over funds and transactions without the need for an intermediary to avoid security risks, hackers or fraud. In a decentralized exchange (DEX), it is very difficult to control the source of money or tax it. The exchange has a system that works by generating proxy tokens (crypto assets representing a currency or cryptocurrency, or generating assets (representing shares in a company), or through an escrow system hierarchical multi-organization - -Keywords: - -- Automatic and direct trading -- Does not depend on 3rd party (intermediaries) -- Avoid risks (security, hacks, scams) - -- **Escrow is** the Investor depositing an amount of money or securities that are accepted **escrow** to secure the obligation to pay for derivative transactions. - -### Why - -**Confidentiality and safety** Depositing funds into the DEX will facilitate trading on the centralized exchange. The number of orders or custody at the exchange will be under the authority of an intermediary trading service i.e. a centralized exchange The security system of DEX is more secure because it does not have a specific fulcrum to create opportunities for hackers to attack. Therefore, it is completely impossible to be attacked by hackers and the floor collapse is very unlikely. - -**Clearness, Authenticity** CEX: the exchange has the ability to manipulate and use users' assets, have the ability to create fake coins, adjust market trends (manipulate) for profit DEX: cannot be manipulated due to blockchain usage (transparent, noneditable) - -**Controls** Transactions are made directly between 2 peers, without intermediaries => managed and controlled directly by users - -**Anonymity** Allow anonymous, no authentication (KYC) - -**Fast transaction** No need to place buy/sell orders and wait for order matching, use [[Smart Contracts]] - ---- - -#### Reference - -https://academy.binance.com/en/articles/what-is-a-decentralized-exchange-dex diff --git a/topics/defi/History.md b/topics/defi/History.md deleted file mode 100644 index 70da6412b..000000000 --- a/topics/defi/History.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: History -description: null -date: null -redirect: - - /C4I18A ---- - -### Bitcoin - -Invented in 2009, starting with Bitcoin, brings distributed ledger/blockchain --> peer-to-peer transfer (P2P) & non-sovereign digital asset (non-sovereign) - -### Ethereum - -Bitcoin - Script language => command restrictions Ethereum, launched in 2015, Solidity language, quickly became a popular smart contract platform because it met the criteria. - -### Maker - -One of the oldest projects on Ethereum is Maker – a protocol for minting the first decentralized stablecoin, DAI. At first, DAI was minted only with ETH as collateral, however the number of accepted collateral expanded in late 2019. - -### EtherDelta decentralized exchange - -Appearing since 2017, EtherDelta allows the exchange of tokens of the ERC20 standard through an order-book mechanism similar to centralized exchanges. Little known due to poor and complicated UX, users mainly use EtherDelta to trade tokens after ICO but not yet listed on centralized exchanges. Hackers gained access to EtherDelta's interface and redirected them to a phishing site – an estimated $800,000 was stolen. - -### ICO - Initial coin offering - -ICO, crowdfunding. Issuing Tokens in exchange for ETH => creating many ghost projects One of the major breakthroughs at the time was the idea of ​​users interacting with smart contracts containing other users' assets, rather than p2p peer-to-peer interaction. This idea created a new “user-to-contract” model, shaping the future of DeFi. After the ICO trend ended, DeFi went through a rather quiet time. - -### Uniswap - -Launched on November 2, 2018, in contrast to EtherDelta, Uniswap is built on completely new concepts such as “liquidity provider”, “automate market maker”. Leverage the previous “user-to-contract” model. - -### Back Thursday - -On March 12, 2020, ETH price plummeted more than 30% in less than 24 hours amid the outbreak of the Corona pandemic. GAS fees increased to 200 gwei as many users tried to deposit more of their collateral to secure various loans or were trying to escape from the bloodbath. Maker is the most affected. => add MKR tokens to compensate. - -### Defi timeline - -May 2020, Compound proposes Liquidity Mining. DeFi users are rewarded for lending and borrowing on Compound. The reward is COMP tokens, resulting in significantly increased lending and borrowing liquidity. This also allows the growth of “Yield Farming” as users can use tokens between protocols to optimize profits. - -New governance model (DAO), users with reward tokens can participate in proposals or vote on proposals with the protocol. - -In early 2020, Yearn launched as one of the tools to optimize productivity by automatically switching between different lending protocols. Andre, the developer of Yearn decided to distribute the governance token – YFI 0 to the Yearn user community in July 2020. Tokens are distributed entirely via liquidity mining – no funds, no home rewards funding, no dev fund. This model has attracted much support from the DeFi community, with cash flows in liquidity pools reaching over $600 million. YFI grew from 6 USD when it was first listed on Uniswap and peaked to over 40000 USD after being listed on Binance. => Many places jump to clone YFI, - -September 11, 2020, YAM was launched. YAM tokens are also distributed in the spirit of YFI's fair launch. It is YAM that is the protocol that helps COMP, LEND(AAVE), LINK, MKR, SNX and YFI projects attract more attention of the crypto community by encouraging them to place these tokens on YAM's platform. . Less than 2 days after launch, a critical bug in the rebase mechanism was found. => Risk perception in prods with high APY - -In August 2020, Sushiswap launched, began to transfer large amounts of liquidity from Uniswap. - -After that, many more clones appeared. - -Last event: Token UNI. All previous users of Uniswap suddenly received an amount of more than 1000 USD (at that time) => regain liquidity from Sushiswap. diff --git "a/topics/defi/NFT/\302\266 NFT.md" "b/topics/defi/NFT/\302\266 NFT.md" deleted file mode 100644 index 24880ab36..000000000 --- "a/topics/defi/NFT/\302\266 NFT.md" +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: ¶ NFT -description: null -date: null -redirect: - - /NjozjA ---- - -NFT is an abbreviation for non-fungible token. It's one the tokens that represents an ownership of rare digital goods such as digital artworks or collectibles. - -In economics, fungibility is the property of a good or a commodity whose individual units are essentially interchangeable and each of whose parts is indistinguishable from another part - -Example: Goods, currency - -![](https://finematics.com/wp-content/uploads/2020/09/nfts-fungibility-2048x1122.png) - -Non-fungible: VD: Original artwork - -Attributes of NFT: - -- Unique: Each NFT is unique and can't be replaced -- Indivisible: NFTs can't be separated into smaller parts with lower valuation. NFTs can't be traded as parts. - -NFTs are minted through smart contracts, in which specifies some attributes of NFTs such as TokenID (unique), owner address and other attributes to help identify an NFT (graphic, image,...). NFT transactions are based on blockchain. Therefore, all the transaction histories of owners are transparency from the beginning to the end. - -Although NFT can be created on any blockchain network that supports smart contracts, but the most common-used networks are [[¶ ERC]]-721 and ERC-1155 on Ethereum - -- The ERC-721 non-fungible token standard is written in Solidity language on the Ethereum blockchain and it allows developers to tokenize ownership of any arbitrary data. -- ERC-1155, an improved standard beyond ERC-721, is another token standard on the Ethereum blockchain that facilitates the creation of both kinds of tokens, fungible and non-fungible. The goal is to create a smart contract interface that can represent both types. - -Storage: - -- NFT is stored on-chain, normally is the blockchain where that NFT first minted. NFT that minted on this blockchain network might not be transferable. -- NFT content (graphic, image,...) is stored off-chain. The data of this NFT will be saved at where the NFT is stored on any server, or IPFS. - - IPFS: (InterPlanetary File System): Decentralized, the data will be encoded and uploaded on other computers. This prevents the server to be disrupted and improves loading performance. It also enables data to be transferred from the current computer to the closest computer. - ---- - -#### Reference - -- https://www.christies.com/features/NFT-101-Collection-Guide-to-NFT-11654-7.aspx -- https://finematics.com/what-are-nfts-and-how-can-they-be-used-in-defi/ diff --git a/topics/defi/Pasted image 20210810212408.png b/topics/defi/Pasted image 20210810212408.png deleted file mode 100644 index e270c4aa7..000000000 Binary files a/topics/defi/Pasted image 20210810212408.png and /dev/null differ diff --git a/topics/defi/Pool.md b/topics/defi/Pool.md deleted file mode 100644 index f48d0e84b..000000000 --- a/topics/defi/Pool.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Pool -description: null -date: null -redirect: - - /daYa1w ---- - -### Liquidity pool - -Liquidity pool: a collection of funds locked in a smart contract. Centralized: When trading, the seller and buyer agree on the same price => Order execution Decentralized: perform transactions without a centralized party to keep money, each order requires gas fees => raise the transaction price Gas fees: fees are given to measure the computational work of transactions, gas prices are determined by miners. In a nutshell, Gas fees are payments made by users in exchange for the computing power needed to process and validate transactions on the Ethereum blockchain. - -### Add liquidity - -Provider creates a pool for an asset pair, eg ETH + USDT. When the Pool is created, the provider will put the same amount of coins in the pool with the same price. For example: 1 ETH = 2484.52 USDT, then the amount of coins put into the pool must correspond. For each pool created, the provider will receive a new Token representing their Share, called Token Pool. The part of the transaction fee will be paid on each Swap transaction, divided by the Shares. For example, if the transaction fee is 0.3%, if the provider has contributed 10% to the ETH+USDT Pool, they will receive 10% of that 0.3%. diff --git a/topics/defi/Swap.md b/topics/defi/Swap.md deleted file mode 100644 index 94b0f9e01..000000000 --- a/topics/defi/Swap.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Swap -description: null -date: null -redirect: - - /6V4NYQ ---- - -### Smart contract - -**Liquidity** A market broker or institution, which behaves as a market maker in a chosen asset class. Act at both ends of currency transactions. he sells and buys a particular asset at certain prices. He is making the market. - -Anything exists because there are buyers and sellers. If you sell without a buyer, you can't match orders. The opposite is true for the buying direction, when you want to buy but no one is selling, the transaction will not take place. If it was simply buyers and sellers transacting with each other without the need for Liquidity Providers, the order matching would take place very slowly, not as quickly as now, simply because it is difficult for buyers and sellers to match orders with each other. perfectly. - -Yield Farming is also known as Liquidity Mining, a way to generate rewards for holding cryptocurrency. In other words, when the Yield Farm process occurs, the user will lock one token and receive a reward in another token. This form is exactly the same as [Staking](https://cryptoviet.com/staking-la-gi). However, Yield Farm works differently from Staking. Specifically, it interacts with Liquidity Providers to provide liquidity to the Liquidity Pools in the protocol. - -**Liquidity provider fee** This fee is split by liquidity providers proportional to their contribution to liquidity reserves. Swapping fees are immediately deposited into liquidity reserves. - -**Route** - -**Price Impact** Gives you an idea what slippage to actually expect based on the size of the order you're placing and what's going on in the marke. This helps you get an idea of how much of the desired token you are likely to actually receive once the trade executes. **Minimum received** - -**Slippahe tolerance** The difference between the theoretical price and the actual receiving price - -### Token list - -ENS = ethereum name service domain host on ethereum chain - -HTTP (HyperText Transfer Protocol): protocol for transferring data from server to web and vice versa, centralized, depends on client-server, if server crashes client will not be able to access, or server holds too much user's data leading to the risk of data leakage (hacking, selling data, ...) - -IPFS: (InterPlanetary File System): decentralized, uploaded data is encrypted and uploaded to other computers participating in the network => no fear of server crash + faster access, can download data from the host machine most recent data. - -ETH token: is a smart contract that runs on ETH to generate tokens diff --git a/topics/defi/Uniswap/Block.md b/topics/defi/Uniswap/Block.md deleted file mode 100644 index 811bb35a3..000000000 --- a/topics/defi/Uniswap/Block.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Block -description: null -date: null -tags: - - block - - blockchain - - defi -redirect: - - /cYD4zQ ---- - -**Block height** aka block number, the length of blockchain **Time stamp** date and time that block is mined **Transaction** number of transctions in the block. - -- 191 transaction and 97 contract internal transaction **Mined by** who include the block onto the blockchain **Block reward** for each block, miner is rewarded with a finite amount of Ether (coin) on top of fee paid for all transactions in the block diff --git a/topics/defi/Uniswap/Chart.md b/topics/defi/Uniswap/Chart.md deleted file mode 100644 index a25a555a2..000000000 --- a/topics/defi/Uniswap/Chart.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: Chart -description: null -date: null -tags: - - blockchain - - defi -redirect: - - /EriDrw ---- - -**TVL** Total value locked: Total amount of money that provider put into trading platforms. **24H volume** the 24-hour trading volume of a cryptocurrency is how much value of a coin has been bought and sold over the course of a day. ![[app.uniswap.org_.png]] diff --git a/topics/defi/Uniswap/DAO.md b/topics/defi/Uniswap/DAO.md deleted file mode 100644 index 2ab073e02..000000000 --- a/topics/defi/Uniswap/DAO.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: DAO -description: null -date: null -tags: - - blockchain - - defi -redirect: - - /aPAPaw ---- - -### Decentralized autonomous organization - -Usually each organization will have a person to make decisions on behalf of, for example, administrators will represent shareholders, politicians will represent citizens. In such organizations, there is often a moral hazard, when rights are vested in the person who brings personal interests above the common good. - -For traditional organizations, the relationship between an organization and an individual is established on an agreement (law, paperwork, tacit agreement), the rights and obligations of each party are regulated and enforced. by a third party or individual. - -A DAO is an organization of a group of people interacting according to a self-enforcing open-source protocol. Keeping the network secure and performing on other networks will earn you tokens of that network. The members of the DAO are not bound to each other by a single legal entity, instead, they are governed by smart contracts - -### How DAO works - -- A group of people will operate this program (based on Smart Contract). -- Next, it's the stage of raising capital by buying shares, also known as ICO (Initial Coin Offering). -- After sufficient funds, the DAO will be put into operation. -- Participants buying shares will have the right to vote on decisions made with the DAO. -- It is important to note that the DAO is not owned by anyone, it is just a piece of software that runs on top of the Ethereum system. People who contribute shares to the DAO will be able to vote on decisions, but do not own the DAO. - -### Public interest - -- From 2020 to 2021, along with the development of Defi, DAO began to be mentioned more. -- Currently DAO has not been recognized by any country and has no clear legal regulations. However, DAOs are widely used in the Defi market, and are recognized in places of legal autonomy (eg. [Wyoming in USA](https://www.cryptopolitan.com/wyoming-recognizes-eos-based-dao/)) - -### DAOs can be used for: - -- Aggregate resources to buy and sell expensive NFT artworks for profit. -- Fund blockchain public goods that are necessary but do not have a business model behind them, with votes issued as donations and donations matched by a pool. -- Essential crowdfunding running on blockchain but no business model, with votes issued as donations and donations combined by a team. Provide policy options on DeFi protocols. -- Accept and reject members who want to join the organization and make decisions in the organization. -- Create talent and non-hierarchical organizations. -- Buy and sell digital assets, such as tokenized real estate or cryptocurrency. - -### Is the world moving towards a DAO model? - -- The legal system began to closely monitor DAOs -- DAO pioneers argue that the importance of DAOs can replace paper organizations, even governments. Regardless of the possibility that governments will soon run countries on-chain, many use cases for DAOs suggest that the ability to combine with other blockchain-based options will greatly contribute to making DAO more popular. . As on-chain solutions and dApps evolve, so will the accessibility and utility of DAOs. - ---- - -#### Reference - -https://d-core.net/decetranlized-autonomous-organizations-explained/ diff --git a/topics/defi/Uniswap/Image/Pasted image 20210623135239.png b/topics/defi/Uniswap/Image/Pasted image 20210623135239.png deleted file mode 100644 index 7d0f2ed53..000000000 Binary files a/topics/defi/Uniswap/Image/Pasted image 20210623135239.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/Pasted image 20210623135240.png b/topics/defi/Uniswap/Image/Pasted image 20210623135240.png deleted file mode 100644 index 7d0f2ed53..000000000 Binary files a/topics/defi/Uniswap/Image/Pasted image 20210623135240.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/app.uniswap.org_ 1.png b/topics/defi/Uniswap/Image/app.uniswap.org_ 1.png deleted file mode 100644 index a6462b2d0..000000000 Binary files a/topics/defi/Uniswap/Image/app.uniswap.org_ 1.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/app.uniswap.org_.png b/topics/defi/Uniswap/Image/app.uniswap.org_.png deleted file mode 100644 index a6462b2d0..000000000 Binary files a/topics/defi/Uniswap/Image/app.uniswap.org_.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/pair chart.png b/topics/defi/Uniswap/Image/pair chart.png deleted file mode 100644 index 16717b41b..000000000 Binary files a/topics/defi/Uniswap/Image/pair chart.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/uniswap-chart-overview.png b/topics/defi/Uniswap/Image/uniswap-chart-overview.png deleted file mode 100644 index 54ec65a1e..000000000 Binary files a/topics/defi/Uniswap/Image/uniswap-chart-overview.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/uniswap-swap-connected.png b/topics/defi/Uniswap/Image/uniswap-swap-connected.png deleted file mode 100644 index dfe89c0d0..000000000 Binary files a/topics/defi/Uniswap/Image/uniswap-swap-connected.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/uniswap-swap-est.png b/topics/defi/Uniswap/Image/uniswap-swap-est.png deleted file mode 100644 index 35e6c28e5..000000000 Binary files a/topics/defi/Uniswap/Image/uniswap-swap-est.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/uniswap-swap-settings.png b/topics/defi/Uniswap/Image/uniswap-swap-settings.png deleted file mode 100644 index 4fe8e2dfb..000000000 Binary files a/topics/defi/Uniswap/Image/uniswap-swap-settings.png and /dev/null differ diff --git a/topics/defi/Uniswap/Image/uniswap-swap.png b/topics/defi/Uniswap/Image/uniswap-swap.png deleted file mode 100644 index 11897d744..000000000 Binary files a/topics/defi/Uniswap/Image/uniswap-swap.png and /dev/null differ diff --git a/topics/defi/Uniswap/Note.md b/topics/defi/Uniswap/Note.md deleted file mode 100644 index bee32e321..000000000 --- a/topics/defi/Uniswap/Note.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: Note -description: null -date: null -tags: - - blockchain - - defi -redirect: - - /M1Tg9g ---- - -### Uniswap - -- Uniswap is a leading decentralized crypto exchange that runs on the Ethereum blockchain. -- DEX, Decentralized exchange, in which, all transactions are made automatically and directly between users (peer-to-peer network) without depending on a third-party. In other way, users have full control of their assets and transactions without rely on any intermediary to avoid ricks, hackers or scams. - -To start to use Uniswap, user have to connect a wallet, which supported Ethereum. ![[uniswap-swap.png]] - -### Pool - -Uniswap works with a model that involves liquidity providers creating liquidity pools. Every transaction requires both seller and buyer. If the user wants to buy but there is no seller, the transaction can not happen, and vice versa. If transactions are made without liquidity pools, the matching order will take a long time, which causes a high difference between the actual price and the estimated one. - -**Liquidity Pool** is a collection of funds locked in a smart contract. Liquidity Provider will place the equal amount of a pair of assets into a Pool, this amount will be locked and unavailable to make other transactions like send or receive. With each pool created, the Provider will receive a new token representing their shares, called Token Pool. The transaction fee will be paid on each Swap transaction, divided by shares. - -![[app.uniswap.org_ 1.png]] - -For example, user want to put 2 ETH into ETH/USDT pool, he has to put both 2 ETH and the corresponding amount of USDT (market price: _2621.15 USDT per ETH_), which means, he has to put **2 ETH** AND **5242,3 USDT** into the pool. The amount that user put into the ETH/USDT pair accounted for about 0.01% of that pool. There is a 0.3% fee for all trades on this pair, the fee will be shared for all providers of this pool, thus, user will earn 0.01% of the 0.3% fee for every ETH/USDT swap transaction. - -### Swap - -**Swap** is the swap of 2 coins on a separate blockchain system, based on Smart Contract, allowing users to trade coins directly from the wallet. - -**Smart Contract** is a self-executing contract with the terms of the agreement between buyer and seller being directly written into lines of code. The code and the agreements contained therein exist across a distributed, decentralized network. The code controls the execution, and transactions are trackable and irreversible. - -![[uniswap-swap-connected.png]] - -![[uniswap-swap-settings.png]] - -**Slippage tolerance**: Acceptance difference between theory price and actual receiving price - -**Transaction deadline**: Transactions are only done in a certain time frame - -**Expect mode**: skip the confirm screen and allow high price impact trades - -**Disable Multihops**: restrict swap to direct pair only - -![[uniswap-swap-est.png]] - -### Chart - -Chart shows all transactions happens on Uniswap and market overview of every token and pair. ![[uniswap-chart-overview.png]] - -**TVL**(Total value locked): amount of money providers put into the exchange. According to the graph, currently there are $1.81B locked in exchange. - -**24H volume**: the 24-hour trading volume of a cryptocurrency is how much value of a coin has been bought and sold over the course of a day. - -**Fees 24H**: Fee of all transaction within 24-hour - -![[pair chart.png]] - -**0.30%** is the fee of liquidity pool, applies for every swap transaction in this pair. There are 3 fee tiers: - -- 0.05% – expected for stablecoin pools like DAI/USDC -- 0.30% – for standard non-correlated pools like ETH/DAI -- 1.00% – for exotic non-correlated pairs - -![[Pasted image 20210623135239.png]] - -**Liquidity chart** - -- X-axis: Asset price -- Y-axis: Liquidity - -For example: Currently (the pink bar), the price of _1 ETH = 2,018.2338 USDT_ and there are 0.64m USDT locked in pool. When the price of _1ETH = 2,790.5834 USDT_ (the selected bar), there were 0.50k ETH locked in pool. diff --git "a/topics/defi/Uniswap/\302\266 Definition.md" "b/topics/defi/Uniswap/\302\266 Definition.md" deleted file mode 100644 index 0173416a9..000000000 --- "a/topics/defi/Uniswap/\302\266 Definition.md" +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Definition -description: null -date: null -tags: - - blockchain - - defi -redirect: - - /ZUu9Hw ---- - -Uniswap is a leading decentralized crypto exchange that runs on the Ethereum blockchain. The vast majority of crypto trading takes place on centralized exchanges such as Coinbase and Binance. These platforms are governed by a single authority (the company that operates the exchange), require users to place funds under their control and use a traditional order book system to facilitate trading. - -Uniswap is a completely different type of exchange that‘s fully [[lynguyen/Uniswap/Decentralized Exchange/Definition]] – meaning it isn’t owned and operated by a single entity – and uses a relatively new type of trading model called an automated liquidity protocol diff --git a/topics/defi/defi-application.md b/topics/defi/defi-application.md deleted file mode 100644 index 9d9fc3fb9..000000000 --- a/topics/defi/defi-application.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: DeFi application -description: null -date: null -redirect: - - /9cNzqA ---- - -## Lending - -DeFi allows any individual to take out or supply a loan without approval from a third party. - -- [AAVE](https://app.aave.com/markets) -- [FulCrum](https://fulcrum.trade/) -- [BlockFi] -- [Compound] -- [Nexo] -- [CoinList] -- [Curve] -- [dYdX] -- [yEarn] -- [Maker] -- [mStable] -- [Nuo Network] - -## Decentralized exchange - -[[¶ Decentralized exchange]] allow users to swap their assets without having to transfer custody of the underlying collateral - -- [0x Protocol] -- [1inch] -- [Bancor] -- [dYdX] -- [Kyber Network] -- [Oasis] -- [Open Sea] -- [Matcha] -- [Uniswap] - -## Derivatives - -In traditional finance, a derivative represents a contract where the value is derived from an agreement based on the performance of an underlying asset. - -- [Augur] -- [dY/dX] -- [Erasure] -- [Opium] -- [Opyn] -- [Synthetix] -- [UMA] - -## Wallet - -Wallets are a crucial gateway for interacting with DeFi products. - -- [Argent] -- [Authereum] -- [Coinbase] -- [Magic] -- [Metamask] -- [Portis] - -## Asset management - -Tools are in place to better track and manage assets - -- [Balancer] -- [Defi Saver] -- [InstaDapp] -- [Sablier] -- [Set Protocol] -- [Zerion] -- [Zapper] - -## Insurance - -Decentralized insurance protocols allow users to take out policies on smart contracts, funds, or any other cryptocurrencies through pooled funds and reserves. - -- [Nexus Multual] -- [Opyn] -- [Opium Finance] - -## Savings - -Offer unique and novel ways to earn a return by saving cryptocurrencies. This differs from lending as there is no borrower on the other side of the table. - -- [DAI Saving Rate] -- [Dharma] -- [Pool Together] diff --git a/topics/defi/dex-market-share.md b/topics/defi/dex-market-share.md deleted file mode 100644 index 39344dda7..000000000 --- a/topics/defi/dex-market-share.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Dex market share -description: null -date: null -redirect: - - /oKx1Dg ---- - -Currently, DEX exchanges are competing for market share in the rapidly growing market. - -Report on 7/7/2021 - -1. Uniswap (V3) - 20.1411% -2. Pancaseswap (V2) - 12.3713% -3. MDEX - 9.7715% -4. MDEX (BSC) - 9.4184% -5. Uniswap (V2) - 6.6117% -6. 1inch Liquidity Protocol - 5.6652% -7. Sushiswap - 5.1303% -8. 1inch Exchange - 2.6461% -9. Perpetual Protocol - 2.0179% -10. Quick Swap - 1.8149% - ---- - -#### Related - -- [[¶ Decentralized Exchange]] - -#### Reference - -- [Decentralized exchanges news 2021](<(https://defirate.com/dex/)>) -- [Top cryptocurrency decentralized exchanges](https://coinmarketcap.com/rankings/exchanges/dex/) diff --git a/topics/defi/how-it-works.md b/topics/defi/how-it-works.md deleted file mode 100644 index b36932b2b..000000000 --- a/topics/defi/how-it-works.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: How it works -description: null -date: null -redirect: - - /8lSqQA ---- - -**Centralized** - -- Deposit your money – either (via bank transfer or credit/debit card) or cryptocurrency => give up control of it. -- Don't own the private key. When user withdraw, user ask the exchange to sign a transaction on their behalves. -- User make transaction: not on chain -- Exchange make a transaction: on chain / allocate balance in its own database => Out of control - -**Decentralized** - -- Orders are executed on-chain with [[Smart Contracts]] and [[AtomicSwap]] -- User do not lost control of their funds diff --git a/topics/defi/liquidity-pool.md b/topics/defi/liquidity-pool.md deleted file mode 100644 index ee5185d45..000000000 --- a/topics/defi/liquidity-pool.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: Liquidity pool -description: null -date: null -author: Hien Le -redirect: - - /ZiMN8w ---- - -## 1. What is a Liquidity Pool? - -Basically, a liquidity pool is a pool of tokens that are locked in a smart contract. Liquidity refers to the ease with which a token can be swapped with another. Anybody can provide liquidity into this single giant pool and earn a share of the trading fees based on their stake in it. - -The process could be illustrated by the picture below: - -![[Pasted image 20220322220453.png]] - -## 2. How do Liquidity Pools work? - -Liquidity pools form the backbone of DEX by applying the automated market maker (AMM) system. Here’s the main formula that mathematically determines what the market price of the token in the pool should be: - -$$ x\*y=k $$ - -x and y represent the respective token balance of a pairing and **k is a constant that will never change**. - -Let’s use the ETH-DAI pair as an example, with 10 ETH and 1,000 DAI in the liquidity pool. What happens when someone wants to buy 1 ETH from this pool? How much does he need to pay? - -The k constant is 10,000 since there are 10 ETH and 1,000 DAI. - -$$ 10 ETH \* 1,000 DAI = 10,000 $$ - -If the buyer withdraws 1 ETH, he has to deposit some DAI into the pool so that k remains constant. - -$$ (10 - 1) ETH \* (1,000 - y) = 10,000$$$$1,000 - y = 10,000/(10-1)$$$$y = 111.11 $$ - -And because we have no limit orders in AMM, the smart contract would automatically compute y to determine the price to pay and that is approximately 111.11 DAI. - -Now the liquidity pool would have 9 ETH and 1,111.11 DAI after someone buys 1 ETH. - -### 2.1. Roles of arbitragers in AMM - -Arbitragers play an important role in AMM. They are used in order to take advantage of the price differences and drive the price back towards market equilibrium. - -If the price of ETH in the pool is higher than it is on Coinbase, arbitragers would sell ETH into this pool and make a profit on the price discrepancies. Thus, the price of ETH in the pool would always be incentivized towards the market price as closely as possible. - -### 2.2. Depth of pool and slippage - -The price difference between the pool and market is known as slippage. How big your slippage is depends on the size of your trade relative to the size or depth of the pool. - -The depth of the pool is measured by that k constant. The bigger your k, the deeper the pool and the less likely a slippage is going to occur. - -In the earlier example, buying 1 ETH from a pool that only has 10 ETH makes up 10% of the pool size. Hence there is such a big difference in price. ETH price costs $100 but you are buying it from the pool at $111.11. That’s about an 11% price slippage. - -In reality, the pool will be much deeper and bigger as there will be hundreds and thousands of liquidity providers from all around the world. - -Suppose we use a pool that has 100 ETH and 10,000 DAI and someone wants to buy 1 ETH from this pool, how much would it cost? Plugging in the same equation would give you $101, a 1% price slippage. - -## 3. How to provide Liquidity into a Pool? - -Anyone can provide liquidity and become a Liquidity Provider (LP). When supplying a pair of tokens into the pool, the ratio price of both tokens must be 50-50. So if you want to provide $5,000 of ETH-DAI pair, you will need $2,500 DAI and $2,500 worth of ETH. - -Every liquidity provider has to follow this standard so that the liquidity pool would always maintain a 50-50 mix of token A and token B. - -When you provide liquidity into a pool, you typically receive an LP token in exchange. This LP token represents your share in the liquidity pool. Every time when a trade is made on the liquidity pool, users have to pay a fee. These fees are then aggregated and re-distributed back to all liquidity providers on a pro-rata basis based on the amount of LP tokens you hold. - -However, you may not get back the exact amount of tokens you deposited initially. That is to say, if you started out with some ETH-DAI tokens, you would get back more ETH and less DAI, or more DAI and less ETH depending on the markets. - -In a bull market, more people would want ETH as prices are rising. Hence the supply of ETH in the pool would drop while DAI would increase since more people are exchanging their dollars for ETH. When you withdraw out your LP, you would end up with less ETH than you started out with and more DAI. The reverse holds true in a bear market. - -## 4. What is Impermanent Loss? - -Impermanent loss refers to the situation where you could have made more if you have done nothing and hold on for dear life rather than providing LP. - -Suppose the price of ETH in our LP is $100. What if the price of ETH on Coinbase rises to $120 in the market? Arbitragers will come in and buy ETH from the pool and sell it on Coinbase to profit from that difference. - -Let’s use a pool that has 100 ETH and 10,000 DAI. The relation between x, y, k, and ETH price could be shown by: - -$$ x _ y = k$$$$x = ETH\ price _ y $$ - -We could easily calculate x and y by k and ETH price: - -$$ x = \sqrt{k \over ETH\ price}$$ -$$y = \sqrt{k \* ETH\ price} $$ - -Assume someone supplies 1 ETH and 100 DAI into the pool. How much ETH and DAI he could get back if the ETH price pumps to $120? - -$$ k = 100 _ 10,000 = 1,000,000 $$$$x = \sqrt{1,000,000 \over 120} = 91.29$$$$y = \sqrt{1,000,000 _ 120} = 10,954.45 $$ - -Since his share in the pool is 1%, the LP gets back 0.9129 ETH and 109.5445 DAI if he wants to withdraw his stake in the pool. The total value of his stake would be 0.9129 ETH \* $120 + $109.54, which totals up to be $219.09. - -If he did not provide his liquidity into the pool and held on for dear life instead, his initial asset would be worth 1 ETH \* $120 + $100 = $220, which also means he would have made an extra $0.91. - -That is what we call impermanent loss. It is impermanent because it only becomes permanent when you withdraw out your LP. - ---- - -#### Reference - -- https://www.jumpstartmag.com/how-do-crypto-liquidity-pools-work/ -- https://www.theancientbabylonians.com/what-is-liquidity-pool-lp-in-defi/#:~:text=To%20sum%20up%20what%20liquidity,%3A%20x%20*%20y%20%3D%20k diff --git a/topics/defi/market-share-dex.md b/topics/defi/market-share-dex.md deleted file mode 100644 index 916c817e1..000000000 --- a/topics/defi/market-share-dex.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: Market share dex -description: null -date: null -redirect: - - /HyNJCw ---- - -In 2020, the market started to develop In 2021, it grew and became the mainstream - -Currently, DEX exchanges are competing for market share in the rapidly growing market. - -### Market share - -https://coinmarketcap.com/rankings/exchanges/dex/ (Number on 7/7/2021) - -1. Uniswap (V3) - 20.1411% -2. Pancaseswap (V2) - 12.3713% -3. MDEX - 9.7715% -4. MDEX (BSC) - 9.4184% -5. Uniswap (V2) - 6.6117% -6. 1inch Liquidity Protocol - 5.6652% -7. Sushiswap - 5.1303% -8. 1inch Exchange - 2.6461% -9. Perpetual Protocol - 2.0179% -10. Quick Swap - 1.8149% - ---- - -#### Reference - -https://defirate.com/dex/ diff --git a/topics/defi/smart-contract.md b/topics/defi/smart-contract.md deleted file mode 100644 index 4f19dbe78..000000000 --- a/topics/defi/smart-contract.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Smart contract -description: null -date: null -redirect: - - /eBsb5A ---- - -## Design by contract - -Also known as **contract programming**, **programming by contract** and **design-by-contract programming**, is an approach for designing software. - -According to Design by Contract theory, a software system can be viewed as a collection of components that interact with each other based on the precise definition of the covenants in the "contract", a conceptual metaphor with the conditions and obligations of business contracts. - -## Smart contract - -Smart Contract is a term that describes a set of special protocols capable of automatically implementing terms and agreements between the parties to the contract (in this case, the computer system) thanks to the support of Blockchain technology. The entire operation of Smart Contract is performed automatically and without outside intervention, or through an intermediary third party. Transactions made using smart contracts are transparent, easily traceable, and cannot be tampered with or reversed. The terms in a Smart Contract are equivalent to a legal contract and are written in the programming language. - -## How it works - -First, the terms of the contract will be written in a programming language, then encrypted and transferred into a block of the Blockchain. After moving into the block, this Smart Contract will be distributed and replicated by the active nodes on that platform. After receiving the deployment order, the contract will be deployed according to the predetermined terms. At the same time, Smart Contract will also automatically check the implementation of the commitments and terms stated in the contract. - ---- - -#### Reference - -- [Design by contract](https://en.wikipedia.org/wiki/Design_by_contract#Description) -- [Smart contract](https://vicogroup.vn/cong-nghe-moi/smart-contract-hop-dong-thong-minh-la-gi-cach-hoat-dong-ung-dung-loi-ich-cua-no-la-gi-22746.html) diff --git a/topics/defi/smart-contracts.md b/topics/defi/smart-contracts.md deleted file mode 100644 index 75b1d4cb1..000000000 --- a/topics/defi/smart-contracts.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Smart contracts -description: null -date: null -redirect: - - /wW6cfA ---- - -### What - -Computer software designed as an automated self-enforcing contract (thực thi tự động), trigger khi có một hoạt động đáp ứng các nhu cầu đã define trước. basically lines of code that execute a specific function once certain conditions are met. If... then... ex: music => nếu có lượt nghe đáp ứng đủ điều kiện (70% thời lượng) => artist nhận tiền - -- Process auto => more efficient, less prone to human error -- No third party, trustless -- Cost effective (?) - -Another area in which smart contracts are suitable is the financial services industry. For instance, the technology may be used to automate the clearing and settlement of trades, the payment of bond coupons, or even the calculation and payout of insurance claims. - -Despite their obvious applications in finance, smart contracts are versatile enough to apply to practically any industry in which funds, digital assets, or any kind of digital information need to be transferred between parties. The equipment leasing industry, for instance, has made extensive real-world use of these contracts to make lease agreements more efficient. - -In the healthcare sector, the technology is being explored as a countermeasure against data manipulation in clinical trials. Smart contracts can even be used to enforce intellectual property agreements by establishing a definitive record of shared ownership rights and allocating all royalties and earnings from pieces of intellectual property accordingly. diff --git "a/topics/defi/\302\266 DeFi (Decentralized Finance).md" "b/topics/defi/\302\266 DeFi (Decentralized Finance).md" deleted file mode 100644 index 3577f43cd..000000000 --- "a/topics/defi/\302\266 DeFi (Decentralized Finance).md" +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: ¶ DeFi (Decentralized Finance) -description: null -date: null -tags: - - blockchain - - defi -redirect: - - /DoT09w ---- - -DeFi is an ecosystem of financial services built on Blockchain technologies, typically using [[Smart Contract]]. DeFi consists of applications and peer-to-peer protocols developed on decentralized blockchain networks that require no access rights for easy lending, borrowing, or trading of financial tools. - -DeFi is built from 3 waves of blockchain innovation: - -1. Invented in 2009, starting with Bitcoin, brings distributed ledger/blockchain --> peer-to-peer transfer (P2P) & non-sovereign digital asset (non-sovereign) -2. Defined by Ethereum, unlike blockchain, ETH's native programming language Solidity can be used to create an application, turning it into a globally accessible supercomputer. -3. Coin boom 2017, marking the sponsorship of a series of [[¶ DApp]] --> decentralized financial ecosystem -4. The 4th wave is Defi, which is a combination of the 3 waves above - -Two main criterias: - -- **Permissionless**: server anyone, regardless of faction, no central authority -- **Transparency**: all documents can be inspected and reviewed because they are open source. Every transaction is recorded on a blockchain -> data discovery for investment or investigation - ---- - -#### Related - -- Blockchain -- [[DeFi Application]] -- [[¶ Decentralized Exchange]] - -#### Reference - -[Decentralized Finance: What It Is, Why It Matters](https://future.a16z.com/cryptos-fourth-wave-defi-poised-for-breakthrough/) diff --git "a/topics/defi/\302\266 DApp.md" "b/topics/defi/\302\266 DApp.md" deleted file mode 100644 index d13fc198d..000000000 --- "a/topics/defi/\302\266 DApp.md" +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: DApps -description: null -date: null -tags: - - blockchain - - dapp -redirect: - - /fLduHg ---- - -### DApps - -![](https://github.com/ethereumbook/ethereumbook/raw/develop/images/web3suite.png) - -#### What is DApp? - -A Dapp is an application that is **mostly** or **entirely** decentralized. The possible aspects of an application that may be decentralized: - -- Backend -- Frontend -- Data storage -- Message communications -- Name resolution - -#### DApp advantages - -- **Resilency**: DApp will have no downtime and will continue to be available as long as the platform is still operating. -- **Transparency**: Everyone can inspect the code. Any interaction with the DApp will be stored forever in the blockchain. -- **Censorship resistance**: No service provider, or even the owner of the smart contract, can alter the code once it is deployed on the network. - -#### The DApp components - -##### Backend (smart contract) - -- You can think of a smart contract replacing a server-side (aka “backend”) component in a regular application. -- Any computation executed in a smart contract is very expensive and so should be kept as minimal as possible. -- Smart contracts in the network can call and pass data between each other, reading and writing their own state variables as they go, with their complexity restricted only by the block gas limit. -- Smart contract architecture design is the inability to change the code of a smart contract once it is deployed. -- _Check gas cost here: [gas cost](https://github.com/djrtwo/evm-opcode-gas-costs/blob/master/opcode-gas-costs_EIP-150_revision-1e18248_2017-04-12.csv)_ - -##### Frontend - -- The client-side interface of a DApp can use standard web technologies (HTML, CSS, JavaScript, etc.) and mobile as well. -- Lnked to Ethereum via the _web3.js_ JavaScript library, which is bundled with the frontend resources and served to a browser by a web server. - -##### Data storage - -- Due to high gas costs and the currently low block gas limit, smart contracts are not well suited to storing or processing large amounts of data. -- Most DApps utilize **off-chain** data storage services, meaning they store the bulky data off the Ethereum chain, on a data storage platform. -- Data storage platform can be centralized (cloud database), or decentralized, stored on a P2P platforms (IPFS, Swarm). - -##### IPFS - -- The _Inter-Planetary File System_ (IPFS) is a decentralized content-addressable storage system that distributes stored objects among peers in a P2P network. -- Content addressable” means that each piece of content (file) is hashed and the hash is used to identify that file. You can then retrieve any file from any IPFS node by requesting it by its hash. - ---- - -#### Reference - -[ethereumbook chap12](https://github.com/ethereumbook/ethereumbook/blob/develop/12dapps.asciidoc) diff --git "a/topics/defi/\302\266 Decentralized Exchange.md" "b/topics/defi/\302\266 Decentralized Exchange.md" deleted file mode 100644 index d2a60cea4..000000000 --- "a/topics/defi/\302\266 Decentralized Exchange.md" +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: ¶ Decentralized exchange -description: null -date: null -redirect: - - /r7AfvQ ---- - -DEX (Decentralized exchange) or decentralized exchange, in which all transactions take place automatically and directly between users (peer-to-peer network), without having to rely on an intermediary to avoid security risks, hackers or scams. - -In CEX (Centralized exchange), when users make transactions, they have to deposit an amount of money (transfer via bank or credit/debit card) or cryptocurrency to the CEX, it means, they give up their control of their assets. When users withdraw, they ask the exchange to sign a transaction on their behalves. CEXs have the ability to manipulate and use users' assets, have the ability to create fake coins, adjust market trends for profit. - -In DEX, orders are executed on-chain with [[Smart Contract]] and [[Atomic Swap]] user do not lost control of their funds - ---- - -#### Related - -- Blockchain -- [[Market share - DEX]] -- [[¶ History of Defi]] -- [[¶ Liquidity Pool]] -- [[Yield Farming]] - -#### Reference - -[# What Is a Decentralized Exchange (DEX)?](https://academy.binance.com/en/articles/what-is-a-decentralized-exchange-dex) diff --git "a/topics/defi/\302\266 ERC.md" "b/topics/defi/\302\266 ERC.md" deleted file mode 100644 index d8b10dc20..000000000 --- "a/topics/defi/\302\266 ERC.md" +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: ¶ ERC -description: null -date: null -redirect: - - /M1GoqA ---- - -In Ethereum, [ERC](https://academy.binance.com/vi/articles/an-introduction-to-erc-20-tokens) stands for _Ethereum Request for Comments_. They are technical documents that outline standards for programming on Ethereum. They are distinguished from Ethereum Improvement Proposals (EIP). EIP, like Bitcoin's BIP, proposes improvements to the protocol itself. - --> ERC aims to establish conventions that make it easier for applications and contracts to interact with each other. diff --git "a/topics/defi/\302\266 History of Defi.md" "b/topics/defi/\302\266 History of Defi.md" deleted file mode 100644 index d97ce8339..000000000 --- "a/topics/defi/\302\266 History of Defi.md" +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: ¶ History of defi -description: null -date: null -redirect: - - /aWOPCA ---- - -### Bitcoin - -Invented in 2009, bắt đầu với Bitcoin, mang đến distributed ledger/blockchain --> peer-to-peer transfer (P2P) & non-sovereign digital asset (không chủ quyền) - -### Ethereum - -Bitcoin - ngôn ngữ Script => hạn chế câu lệnh Ethereum, ra mắt 2015, ngôn ngữ Solidity, nhanh chóng trở thành một nền tảng hợp đồng thông minh phổ biến  vì đáp ứng được các tiêu chí. - -### Maker - -Một trong những dự án lâu đời nhất trên Ethereum là Maker – một giao thức cho phép đúc ra stablecoin phi tập trung đầu tiên, DAI. Lúc đầu, DAI chỉ được đúc bằng tài sản thế chấp là ETH, tuy nhiên số lượng tài sản thế chấp được chấp nhận đã mở rộng vào cuối năm 2019. - -### EtherDelta decentralized exchange - -Xuất hiện từ năm 2017, EtherDelta cho phép trao đổi các token thuộc chuẩn ERC20 thông qua cơ chế order-book tương tự các sàn tập trung. Ít người biết đến do UX kém và phức tạp, người dùng chủ yếu sử dụng EtherDelta để giao dịch các token sau khi ICO nhưng chưa được niêm yết trên các sàn giao dịch tập trung. Hacker giành được quyền truy cập vào giao diện của EtherDelta và redirect họ đến một trang web lừa đảo – ước tính 800 nghìn USD đã bị đánh cắp. - -### ICO - Initial coin offering - -ICO, gọi vốn cộng đồng. Phát hành Token để đổi lấy ETH => tạo nên nhiều dự án ma Một trong những đột phá chính vào thời điểm đó là ý tưởng về việc người dùng tương tác với các hợp đồng thông minh chứa tài sản của những người dùng khác, thay vì tương tác ngang hàng p2p. Ý tưởng này đã tạo ra một mô hình mới “user-to-contract”, định hình DeFi trong tương lai. Sau đó trend ICO kết thúc, DeFi trải qua thời gian khá im hơi lặng tiếng - -### Uniswap - -Ra mắt vào ngày 2 tháng 11 năm 2018, trái ngược với EtherDelta, Uniswap được xây dựng dựa trên các khái niệm hoàn toàn mới như “liquidity provider” , “automate market maker”. Tận dụng mô hình “user-to-contract” trước đó. - -### Back Thursday - -Ngày 12 tháng 3 năm 2020, giá ETH giảm mạnh hơn 30% trong vòng chưa đầy 24h trong bối cảnh đại dịch Corona bùng phát. Phí GAS tăng đến 200 gwei do nhiều người dùng cố gắng gửi thêm tài sản thế chấp của họ để bảo đảm các khoản vay khác nhau hoặc đang cố gắng tháo chạy khỏi bể máu. Maker chịu ảnh hưởng nhiều nhất. => bổ sung thêm token MKR để bù đắp. - -### Defi timeline - -Tháng 5 năm 2020, Compound đề xuất Liquidity Mining. Người dùng DeFi được thưởng khi cho vay và vay trên Compound. Phần thưởng là token COMP, dẫn đến việc cung cấp thanh khoản vay và cho vay tăng lên đáng kể. Điều này cũng cho phép sự phát triển của “Yield Farming” vì người dùng có thể sử dụng các token giữa các giao thức để tối ưu hóa lợi nhuận. - -Mô hình quản trị mới (DAO), người dùng có các token thưởng có thể tham gia đề xuất hoặc bỏ phiếu cho các đề xuất với giao thức. - -Đầu năm 2020, Yearn ra mắt là một trong những công cụ tối ưu hóa năng suất bằng việc tự động chuyển đổi giữa các giao thức cho vay khác nhau. Andre, nhà phát triển Yearn đã quyết định phân phối token quản trị – YFI 0 cho cộng đồng sử dụng Yearn vào tháng 7 năm 2020. Token được phân phối hoàn toàn qua liquidity mining – không có các quỹ, không có phần thưởng cho nhà tài trợ, không có dev fund. Mô hình này đã thu hút nhiều sự ủng hộ từ cộng đồng DeFi, với dòng tiền trong các nhóm thanh khoản đạt đến mức hơn 600 triệu USD. YFI tăng từ 6 USD khi mới được niêm yết trên Uniswap và đỉnh điểm là hơn 40000 USD sau khi được niêm yết lên Binance. => Nhiều nơi nhảy vào clone YFI - -Ngày 11/9/2020, ra mắt YAM. Token YAM cũng được phân phối theo tinh thần fair launch của YFI. Chính YAM là giao thức giúp các dự án COMP, LEND(AAVE), LINK, MKR, SNX và YFI thu hút được sự chú ý của cộng đồng tiền điện tử nhiều hơn khi khuyến khích họ đặt các token này trên nền tảng của YAM. Chưa đầy 2 ngày sau khi ra mắt, một lỗi nghiêm trọng trong cơ chế rebase được tìm thấy. => Nhận thức rủi ro ở những prod có APY cao - -Tháng 8 năm 2020, Sushiswap ra mắt, bắt đầu chuyển số lượng lớn liquidity từ Uniswap sang. - -Sau đó cũng xuất hiện thêm nhiều bản clone khác. - -Sự kiện cuối cùng: Token UNI. Tất cả người dùng trước đây của Uniswap bỗng nhận được một khoản tiền trị giá hơn 1000 USD (tại thời điểm đó) => giành lại liquidity từ Sushiswap - ---- - -#### Related - -- [[¶ DeFi (Decentralized Finance)]] -- [[¶ Decentralized Exchange]] - -#### Reference - -[History Of DeFi – From Inception To 2021 And Beyond](https://finematics.com/history-of-defi-explained/) diff --git "a/topics/defi/\302\266 Liquidity Pool.md" "b/topics/defi/\302\266 Liquidity Pool.md" deleted file mode 100644 index 2831cc112..000000000 --- "a/topics/defi/\302\266 Liquidity Pool.md" +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: ¶ Liquidity pool -description: null -date: null -redirect: - - /B80GyA ---- - -**Liquidity Pool** is a collection of funds locked in a smart contract. Liquidity Provider will place the equal amount of a pair of assets into a Pool, this amount will be locked and unavailable to make other transactions like send or receive. With each pool created, the Provider will receive a new token representing their shares, called Token Pool. The transaction fee will be paid on each Swap transaction, divided by shares. - -For example, a provider wants to put 2 ETH into ETH/USDT pool, he has to put both 2 ETH and the corresponding amount of USDT (market price: _2621.15 USDT per ETH_), which means, he has to put **2 ETH** AND **5242,3 USDT** into the pool. The amount that provider put into the ETH/USDT pair accounted for about 0.01% of that pool. There is a 0.3% fee for all trades on this pair, the fee will be shared for all providers of this pool, thus, the provider will earn 0.01% of the 0.3% fee for every ETH/USDT swap transaction. - ---- - -#### Related - -- [[¶ Decentralized Exchange]] -- [[Atomic Swap]] diff --git a/topics/delegation/.config.yml b/topics/delegation/.config.yml deleted file mode 100644 index fa145e37d..000000000 --- a/topics/delegation/.config.yml +++ /dev/null @@ -1,2 +0,0 @@ -aliases: - - delegation diff --git a/topics/delegation/README.md b/topics/delegation/README.md deleted file mode 100644 index 89665accb..000000000 --- a/topics/delegation/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: § Delegation -description: null -date: null -authors: - - tieubao -redirect: - - /sC62JA ---- diff --git a/topics/design/Human needs.md b/topics/design/Human needs.md deleted file mode 100644 index 56164c6d2..000000000 --- a/topics/design/Human needs.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - design - - ux -redirect: - - /6WN4lQ ---- - -We had already known about the software development process which helped all team members have an overview of building a product. To get a better insight into startup CEO/founders' incentive, we continued to learn about where ideas come from, why people want to bring an idea into reality, and what a good idea is. - -In the kickoff meeting before initiating a project, we often hear that the incentive of making a product is to realize a problem or problems of a group of users and they (startup) want to solve them. In another way, the idea usually comes from human needs or demands. In order to better understand human needs, we went through some popular frameworks that are used in many successful products. - -- [[Maslow's hierarchy of needs]] -- [[Seven deadly sins]] -- [[Skandhas - the five aggregates]] - -Applying this framework to product design, a product not only meets the needs of users, but also must satisfy all human components, both physically, mentally, and emotionally, and consciousness. During the journey of users to accomplish their task, besides emotion, we should evaluate all components, and transform the negative factors into the positive ones. - -These frameworks is used to analyze human needs. Basically, ideas come from responding to human needs to bring back some value (material or non-material). Before an idea comes into play, it is important to carefully measure and evaluate the responsiveness to the needs and its value. - ---- - -#### Related - -- [[Human needs]] -- [[Seven deadly sins]] -- [[Skandhas - the five aggregates]] -- [[Value chain]] diff --git a/topics/design/assets/design-better-mobile-application_3db029547cf3f0df87733d42e499f2b5_md5.webp b/topics/design/assets/design-better-mobile-application_3db029547cf3f0df87733d42e499f2b5_md5.webp deleted file mode 100644 index 00ad8551f..000000000 Binary files a/topics/design/assets/design-better-mobile-application_3db029547cf3f0df87733d42e499f2b5_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-better-mobile-application_56fa2f8d701c84b46a715ca629e77f27_md5.webp b/topics/design/assets/design-better-mobile-application_56fa2f8d701c84b46a715ca629e77f27_md5.webp deleted file mode 100644 index 760076dbe..000000000 Binary files a/topics/design/assets/design-better-mobile-application_56fa2f8d701c84b46a715ca629e77f27_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-better-mobile-application_7c641059d4d0fad9e7fd2c85c45edbaa_md5.webp b/topics/design/assets/design-better-mobile-application_7c641059d4d0fad9e7fd2c85c45edbaa_md5.webp deleted file mode 100644 index fea536ee4..000000000 Binary files a/topics/design/assets/design-better-mobile-application_7c641059d4d0fad9e7fd2c85c45edbaa_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-resourcestools_1d108bd8a65ff50110c20d550f63945e_md5.webp b/topics/design/assets/design-resourcestools_1d108bd8a65ff50110c20d550f63945e_md5.webp deleted file mode 100644 index 4a78e7e6e..000000000 Binary files a/topics/design/assets/design-resourcestools_1d108bd8a65ff50110c20d550f63945e_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-resourcestools_22750df99a8ae9d95127999f8d41614b_md5.webp b/topics/design/assets/design-resourcestools_22750df99a8ae9d95127999f8d41614b_md5.webp deleted file mode 100644 index 0752f5575..000000000 Binary files a/topics/design/assets/design-resourcestools_22750df99a8ae9d95127999f8d41614b_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-resourcestools_35a07daafda041800eb792f9dfb52744_md5.webp b/topics/design/assets/design-resourcestools_35a07daafda041800eb792f9dfb52744_md5.webp deleted file mode 100644 index 8680e0010..000000000 Binary files a/topics/design/assets/design-resourcestools_35a07daafda041800eb792f9dfb52744_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-resourcestools_9863db6fd97f581087152cab779966e2_md5.webp b/topics/design/assets/design-resourcestools_9863db6fd97f581087152cab779966e2_md5.webp deleted file mode 100644 index eebf11ba5..000000000 Binary files a/topics/design/assets/design-resourcestools_9863db6fd97f581087152cab779966e2_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_0018d0d34bb133667d90e34ce27c5fd0_md5.webp b/topics/design/assets/design-system_0018d0d34bb133667d90e34ce27c5fd0_md5.webp deleted file mode 100644 index f0bedb29d..000000000 Binary files a/topics/design/assets/design-system_0018d0d34bb133667d90e34ce27c5fd0_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_29a2cbbe18c551fb6c02d550c7eba526_md5.webp b/topics/design/assets/design-system_29a2cbbe18c551fb6c02d550c7eba526_md5.webp deleted file mode 100644 index 70e5d4ffe..000000000 Binary files a/topics/design/assets/design-system_29a2cbbe18c551fb6c02d550c7eba526_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_3e3694f6596e6a67a2a10b4bb6ea8eed_md5.webp b/topics/design/assets/design-system_3e3694f6596e6a67a2a10b4bb6ea8eed_md5.webp deleted file mode 100644 index 0ee274512..000000000 Binary files a/topics/design/assets/design-system_3e3694f6596e6a67a2a10b4bb6ea8eed_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_777f120275d29fe97d756a8d98d5be91_md5.webp b/topics/design/assets/design-system_777f120275d29fe97d756a8d98d5be91_md5.webp deleted file mode 100644 index b82dda922..000000000 Binary files a/topics/design/assets/design-system_777f120275d29fe97d756a8d98d5be91_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_a9cdca95e3ffef0e55cd90fdec84527a_md5.webp b/topics/design/assets/design-system_a9cdca95e3ffef0e55cd90fdec84527a_md5.webp deleted file mode 100644 index 836a71175..000000000 Binary files a/topics/design/assets/design-system_a9cdca95e3ffef0e55cd90fdec84527a_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_b0d28b9d30e9d838276303ad69f6e89a_md5.webp b/topics/design/assets/design-system_b0d28b9d30e9d838276303ad69f6e89a_md5.webp deleted file mode 100644 index aa086ced2..000000000 Binary files a/topics/design/assets/design-system_b0d28b9d30e9d838276303ad69f6e89a_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_b187e07f45c64d9eabd7e524b39a31ed_md5.webp b/topics/design/assets/design-system_b187e07f45c64d9eabd7e524b39a31ed_md5.webp deleted file mode 100644 index f977d4bce..000000000 Binary files a/topics/design/assets/design-system_b187e07f45c64d9eabd7e524b39a31ed_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_cfe9acf6fd2d1adde475eac22289689b_md5.webp b/topics/design/assets/design-system_cfe9acf6fd2d1adde475eac22289689b_md5.webp deleted file mode 100644 index f035231ac..000000000 Binary files a/topics/design/assets/design-system_cfe9acf6fd2d1adde475eac22289689b_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-system_d4b5482e66a2d78fe46a315c5d2a4646_md5.webp b/topics/design/assets/design-system_d4b5482e66a2d78fe46a315c5d2a4646_md5.webp deleted file mode 100644 index 3942f01c0..000000000 Binary files a/topics/design/assets/design-system_d4b5482e66a2d78fe46a315c5d2a4646_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-token.png b/topics/design/assets/design-token.png deleted file mode 100644 index 904c7a859..000000000 Binary files a/topics/design/assets/design-token.png and /dev/null differ diff --git a/topics/design/assets/design-workflow_1ce44520b17bffd06b033406e0d794a6_md5.webp b/topics/design/assets/design-workflow_1ce44520b17bffd06b033406e0d794a6_md5.webp deleted file mode 100644 index 761f5b988..000000000 Binary files a/topics/design/assets/design-workflow_1ce44520b17bffd06b033406e0d794a6_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-workflow_a19657465a449ff547f2c9008bbfab8c_md5.webp b/topics/design/assets/design-workflow_a19657465a449ff547f2c9008bbfab8c_md5.webp deleted file mode 100644 index eee64fa0f..000000000 Binary files a/topics/design/assets/design-workflow_a19657465a449ff547f2c9008bbfab8c_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-workflow_aad2cd76d88ab384bd09ba16793dab87_md5.webp b/topics/design/assets/design-workflow_aad2cd76d88ab384bd09ba16793dab87_md5.webp deleted file mode 100644 index e7b07c462..000000000 Binary files a/topics/design/assets/design-workflow_aad2cd76d88ab384bd09ba16793dab87_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-workflow_d6a68d2878bdf01a430afc77624683a2_md5.webp b/topics/design/assets/design-workflow_d6a68d2878bdf01a430afc77624683a2_md5.webp deleted file mode 100644 index e8bf6b7e6..000000000 Binary files a/topics/design/assets/design-workflow_d6a68d2878bdf01a430afc77624683a2_md5.webp and /dev/null differ diff --git a/topics/design/assets/design-workflow_d8fe23347d8bff5a0be6ae56bd0fc438_md5.webp b/topics/design/assets/design-workflow_d8fe23347d8bff5a0be6ae56bd0fc438_md5.webp deleted file mode 100644 index bd1b50287..000000000 Binary files a/topics/design/assets/design-workflow_d8fe23347d8bff5a0be6ae56bd0fc438_md5.webp and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-discord.png b/topics/design/assets/designing-for-forgiveness-discord.png deleted file mode 100644 index b5854a372..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-discord.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-drive.png b/topics/design/assets/designing-for-forgiveness-drive.png deleted file mode 100644 index 1bc86b2bf..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-drive.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-excel-setting.png b/topics/design/assets/designing-for-forgiveness-excel-setting.png deleted file mode 100644 index 873c6cf69..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-excel-setting.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-excel.png b/topics/design/assets/designing-for-forgiveness-excel.png deleted file mode 100644 index e5f0fb506..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-excel.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-microsoft.png b/topics/design/assets/designing-for-forgiveness-microsoft.png deleted file mode 100644 index 7061ed641..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-microsoft.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-shopify.png b/topics/design/assets/designing-for-forgiveness-shopify.png deleted file mode 100644 index 2ed064b4f..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-shopify.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-twitter.png b/topics/design/assets/designing-for-forgiveness-twitter.png deleted file mode 100644 index 6762977da..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-twitter.png and /dev/null differ diff --git a/topics/design/assets/designing-for-forgiveness-webflow.png b/topics/design/assets/designing-for-forgiveness-webflow.png deleted file mode 100644 index 3007f5070..000000000 Binary files a/topics/design/assets/designing-for-forgiveness-webflow.png and /dev/null differ diff --git a/topics/design/assets/domain-glossary_4bc9024bc646f7b46a1675cd62ff9350_md5.webp b/topics/design/assets/domain-glossary_4bc9024bc646f7b46a1675cd62ff9350_md5.webp deleted file mode 100644 index 701243de2..000000000 Binary files a/topics/design/assets/domain-glossary_4bc9024bc646f7b46a1675cd62ff9350_md5.webp and /dev/null differ diff --git a/topics/design/assets/domain-glossary_9c8390ea196dd16d4d3dd5a56311ba94_md5.webp b/topics/design/assets/domain-glossary_9c8390ea196dd16d4d3dd5a56311ba94_md5.webp deleted file mode 100644 index bca72efde..000000000 Binary files a/topics/design/assets/domain-glossary_9c8390ea196dd16d4d3dd5a56311ba94_md5.webp and /dev/null differ diff --git a/topics/design/assets/domain-insight-research-framework_212c5cd2b79ff8385855412c63292329_md5.webp b/topics/design/assets/domain-insight-research-framework_212c5cd2b79ff8385855412c63292329_md5.webp deleted file mode 100644 index 3f56d3b59..000000000 Binary files a/topics/design/assets/domain-insight-research-framework_212c5cd2b79ff8385855412c63292329_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_0072fd89ad3c95ef57fb09a6299464ff_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_0072fd89ad3c95ef57fb09a6299464ff_md5.webp deleted file mode 100644 index 399df352f..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_0072fd89ad3c95ef57fb09a6299464ff_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_13186fb397ff23cfe63db16a8ece4846_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_13186fb397ff23cfe63db16a8ece4846_md5.webp deleted file mode 100644 index b8eded0e4..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_13186fb397ff23cfe63db16a8ece4846_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_261d02efd9a855cdb1eed8a98112d3ae_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_261d02efd9a855cdb1eed8a98112d3ae_md5.webp deleted file mode 100644 index 2f42d32a8..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_261d02efd9a855cdb1eed8a98112d3ae_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_3a01b5fecbaf061c21599080993c6400_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_3a01b5fecbaf061c21599080993c6400_md5.webp deleted file mode 100644 index 9745de238..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_3a01b5fecbaf061c21599080993c6400_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_61289ca41b3b2f8dc1cae3cb3b9da6a7_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_61289ca41b3b2f8dc1cae3cb3b9da6a7_md5.webp deleted file mode 100644 index 9d5b097ed..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_61289ca41b3b2f8dc1cae3cb3b9da6a7_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_c25b23553a4fbc3e63707bb9a81d72c4_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_c25b23553a4fbc3e63707bb9a81d72c4_md5.webp deleted file mode 100644 index f964c57c4..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_c25b23553a4fbc3e63707bb9a81d72c4_md5.webp and /dev/null differ diff --git a/topics/design/assets/gestalt-principles-in-ui-design_e25d1641f846de4167a75b8640b83744_md5.webp b/topics/design/assets/gestalt-principles-in-ui-design_e25d1641f846de4167a75b8640b83744_md5.webp deleted file mode 100644 index 9adcb0dd0..000000000 Binary files a/topics/design/assets/gestalt-principles-in-ui-design_e25d1641f846de4167a75b8640b83744_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_05a4d54c84aaf4bbee7fca8473887937_md5.webp b/topics/design/assets/getting-started-with-webflow_05a4d54c84aaf4bbee7fca8473887937_md5.webp deleted file mode 100644 index ebb53c346..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_05a4d54c84aaf4bbee7fca8473887937_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_15ed261870e05e4d45159ac062d63f5c_md5.webp b/topics/design/assets/getting-started-with-webflow_15ed261870e05e4d45159ac062d63f5c_md5.webp deleted file mode 100644 index d8f9f8b81..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_15ed261870e05e4d45159ac062d63f5c_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_2259389132637c4db847b8143770adf4_md5.webp b/topics/design/assets/getting-started-with-webflow_2259389132637c4db847b8143770adf4_md5.webp deleted file mode 100644 index a94b31761..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_2259389132637c4db847b8143770adf4_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_435ece2b3f4f240af0512011540e7010_md5.webp b/topics/design/assets/getting-started-with-webflow_435ece2b3f4f240af0512011540e7010_md5.webp deleted file mode 100644 index 1a05891ff..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_435ece2b3f4f240af0512011540e7010_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_483a7775cfee79b4bfb0d30f3cc2e5ea_md5.webp b/topics/design/assets/getting-started-with-webflow_483a7775cfee79b4bfb0d30f3cc2e5ea_md5.webp deleted file mode 100644 index 4810ca420..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_483a7775cfee79b4bfb0d30f3cc2e5ea_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_523f7073ae646470be25042035c8e949_md5.webp b/topics/design/assets/getting-started-with-webflow_523f7073ae646470be25042035c8e949_md5.webp deleted file mode 100644 index c645d9aaa..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_523f7073ae646470be25042035c8e949_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_5386df53f5360ba922dbe2c4b467dfdb_md5.webp b/topics/design/assets/getting-started-with-webflow_5386df53f5360ba922dbe2c4b467dfdb_md5.webp deleted file mode 100644 index 24b3b8960..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_5386df53f5360ba922dbe2c4b467dfdb_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_683658131c557165d9023be3bbb4cf28_md5.webp b/topics/design/assets/getting-started-with-webflow_683658131c557165d9023be3bbb4cf28_md5.webp deleted file mode 100644 index 4fbbb6565..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_683658131c557165d9023be3bbb4cf28_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_73ebf9dadeae2d52ba898a3fe0de296a_md5.webp b/topics/design/assets/getting-started-with-webflow_73ebf9dadeae2d52ba898a3fe0de296a_md5.webp deleted file mode 100644 index 75753c341..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_73ebf9dadeae2d52ba898a3fe0de296a_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_868bfa2a53c561cff2296e7b08665242_md5.webp b/topics/design/assets/getting-started-with-webflow_868bfa2a53c561cff2296e7b08665242_md5.webp deleted file mode 100644 index ba6d45f0d..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_868bfa2a53c561cff2296e7b08665242_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_9ff9430476ddca3d674f41a53ec439ca_md5.webp b/topics/design/assets/getting-started-with-webflow_9ff9430476ddca3d674f41a53ec439ca_md5.webp deleted file mode 100644 index 1cd7d084a..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_9ff9430476ddca3d674f41a53ec439ca_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_aa6335d35ece7aa09c9484559131c433_md5.webp b/topics/design/assets/getting-started-with-webflow_aa6335d35ece7aa09c9484559131c433_md5.webp deleted file mode 100644 index 0facdb936..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_aa6335d35ece7aa09c9484559131c433_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_ab1951972be3533d03fe024933236f17_md5.webp b/topics/design/assets/getting-started-with-webflow_ab1951972be3533d03fe024933236f17_md5.webp deleted file mode 100644 index 527f2d399..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_ab1951972be3533d03fe024933236f17_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_ad25c3727e62ea25c77238725e166cf3_md5.webp b/topics/design/assets/getting-started-with-webflow_ad25c3727e62ea25c77238725e166cf3_md5.webp deleted file mode 100644 index faa14c4fd..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_ad25c3727e62ea25c77238725e166cf3_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_b7f3ef4d478ea5cb2f99e218854ea861_md5.webp b/topics/design/assets/getting-started-with-webflow_b7f3ef4d478ea5cb2f99e218854ea861_md5.webp deleted file mode 100644 index 896067061..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_b7f3ef4d478ea5cb2f99e218854ea861_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_c1035a354ff1b911ba88d98c7815ad41_md5.webp b/topics/design/assets/getting-started-with-webflow_c1035a354ff1b911ba88d98c7815ad41_md5.webp deleted file mode 100644 index 85386c3cf..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_c1035a354ff1b911ba88d98c7815ad41_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_d458119ad0184eee89c1d611a1543c50_md5.webp b/topics/design/assets/getting-started-with-webflow_d458119ad0184eee89c1d611a1543c50_md5.webp deleted file mode 100644 index c3554f017..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_d458119ad0184eee89c1d611a1543c50_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_f8d5b754890ffc243e735a6e4fb21394_md5.webp b/topics/design/assets/getting-started-with-webflow_f8d5b754890ffc243e735a6e4fb21394_md5.webp deleted file mode 100644 index 889b9c1ac..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_f8d5b754890ffc243e735a6e4fb21394_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_f8fbcb003ce6c8b919bc0cc7235b3110_md5.webp b/topics/design/assets/getting-started-with-webflow_f8fbcb003ce6c8b919bc0cc7235b3110_md5.webp deleted file mode 100644 index c4ea6537c..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_f8fbcb003ce6c8b919bc0cc7235b3110_md5.webp and /dev/null differ diff --git a/topics/design/assets/getting-started-with-webflow_fe50e0a67aa39fa34b8ba639a068d7cf_md5.webp b/topics/design/assets/getting-started-with-webflow_fe50e0a67aa39fa34b8ba639a068d7cf_md5.webp deleted file mode 100644 index 0393a4de3..000000000 Binary files a/topics/design/assets/getting-started-with-webflow_fe50e0a67aa39fa34b8ba639a068d7cf_md5.webp and /dev/null differ diff --git a/topics/design/assets/good-design-understanding_77e7ee12b43f15d452c133150959dd8f_md5.webp b/topics/design/assets/good-design-understanding_77e7ee12b43f15d452c133150959dd8f_md5.webp deleted file mode 100644 index 688129624..000000000 Binary files a/topics/design/assets/good-design-understanding_77e7ee12b43f15d452c133150959dd8f_md5.webp and /dev/null differ diff --git a/topics/design/assets/good-design-understanding_e474e00ace9a524ebdff60d52f5e944c_md5.webp b/topics/design/assets/good-design-understanding_e474e00ace9a524ebdff60d52f5e944c_md5.webp deleted file mode 100644 index edbb76355..000000000 Binary files a/topics/design/assets/good-design-understanding_e474e00ace9a524ebdff60d52f5e944c_md5.webp and /dev/null differ diff --git a/topics/design/assets/good-design-understanding_e75aeb9b90122499635bddc51805aca3_md5.webp b/topics/design/assets/good-design-understanding_e75aeb9b90122499635bddc51805aca3_md5.webp deleted file mode 100644 index d2e8768c7..000000000 Binary files a/topics/design/assets/good-design-understanding_e75aeb9b90122499635bddc51805aca3_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_024c62923ce47df299bd7ee36aa26a77_md5.webp b/topics/design/assets/grid-and-layout_024c62923ce47df299bd7ee36aa26a77_md5.webp deleted file mode 100644 index 40a09fd76..000000000 Binary files a/topics/design/assets/grid-and-layout_024c62923ce47df299bd7ee36aa26a77_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_1d7ff36ad43140f04c22a1344327120a_md5.webp b/topics/design/assets/grid-and-layout_1d7ff36ad43140f04c22a1344327120a_md5.webp deleted file mode 100644 index bc612eaa0..000000000 Binary files a/topics/design/assets/grid-and-layout_1d7ff36ad43140f04c22a1344327120a_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_36c8e99a111f6eed2d177b9b0eb8228f_md5.webp b/topics/design/assets/grid-and-layout_36c8e99a111f6eed2d177b9b0eb8228f_md5.webp deleted file mode 100644 index 80babb152..000000000 Binary files a/topics/design/assets/grid-and-layout_36c8e99a111f6eed2d177b9b0eb8228f_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_3bdce6d13717bd0834f92c4a79970069_md5.webp b/topics/design/assets/grid-and-layout_3bdce6d13717bd0834f92c4a79970069_md5.webp deleted file mode 100644 index a5f9fc91b..000000000 Binary files a/topics/design/assets/grid-and-layout_3bdce6d13717bd0834f92c4a79970069_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_4052e7ff934548603afc4e9abb43548d_md5.webp b/topics/design/assets/grid-and-layout_4052e7ff934548603afc4e9abb43548d_md5.webp deleted file mode 100644 index ed0b8ba2b..000000000 Binary files a/topics/design/assets/grid-and-layout_4052e7ff934548603afc4e9abb43548d_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_7417c8675f9f592d8b8fee0d635848e8_md5.webp b/topics/design/assets/grid-and-layout_7417c8675f9f592d8b8fee0d635848e8_md5.webp deleted file mode 100644 index 806f32f2f..000000000 Binary files a/topics/design/assets/grid-and-layout_7417c8675f9f592d8b8fee0d635848e8_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_7df1c65ee21b0bfe2452413491f8f9bf_md5.webp b/topics/design/assets/grid-and-layout_7df1c65ee21b0bfe2452413491f8f9bf_md5.webp deleted file mode 100644 index 11eb46f0e..000000000 Binary files a/topics/design/assets/grid-and-layout_7df1c65ee21b0bfe2452413491f8f9bf_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_910a469e5bc586180f41f88680bba4bd_md5.webp b/topics/design/assets/grid-and-layout_910a469e5bc586180f41f88680bba4bd_md5.webp deleted file mode 100644 index 58d49c6bd..000000000 Binary files a/topics/design/assets/grid-and-layout_910a469e5bc586180f41f88680bba4bd_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_99ba657b5b121158b61df8f51c3935df_md5.webp b/topics/design/assets/grid-and-layout_99ba657b5b121158b61df8f51c3935df_md5.webp deleted file mode 100644 index 062de7654..000000000 Binary files a/topics/design/assets/grid-and-layout_99ba657b5b121158b61df8f51c3935df_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_a043920fee14ac284854524a4e074fd1_md5.webp b/topics/design/assets/grid-and-layout_a043920fee14ac284854524a4e074fd1_md5.webp deleted file mode 100644 index 9bef5f3cb..000000000 Binary files a/topics/design/assets/grid-and-layout_a043920fee14ac284854524a4e074fd1_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_a106abd338fb3f0957d15d343faef1c5_md5.webp b/topics/design/assets/grid-and-layout_a106abd338fb3f0957d15d343faef1c5_md5.webp deleted file mode 100644 index 89beeb5b8..000000000 Binary files a/topics/design/assets/grid-and-layout_a106abd338fb3f0957d15d343faef1c5_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_c7cd9400e4b6a7c96f2df3c7f24791f9_md5.webp b/topics/design/assets/grid-and-layout_c7cd9400e4b6a7c96f2df3c7f24791f9_md5.webp deleted file mode 100644 index 71198702d..000000000 Binary files a/topics/design/assets/grid-and-layout_c7cd9400e4b6a7c96f2df3c7f24791f9_md5.webp and /dev/null differ diff --git a/topics/design/assets/grid-and-layout_f0045709ef2c31affbb508461c5229cd_md5.webp b/topics/design/assets/grid-and-layout_f0045709ef2c31affbb508461c5229cd_md5.webp deleted file mode 100644 index ac2630cc1..000000000 Binary files a/topics/design/assets/grid-and-layout_f0045709ef2c31affbb508461c5229cd_md5.webp and /dev/null differ diff --git a/topics/design/assets/how-a-design-system-work_462d264e13a03129c48869ecadc606ed_md5.webp b/topics/design/assets/how-a-design-system-work_462d264e13a03129c48869ecadc606ed_md5.webp deleted file mode 100644 index ea9e28230..000000000 Binary files a/topics/design/assets/how-a-design-system-work_462d264e13a03129c48869ecadc606ed_md5.webp and /dev/null differ diff --git a/topics/design/assets/how-a-design-system-work_eb3f1f53df6e5919ae30528c764a27ee_md5.webp b/topics/design/assets/how-a-design-system-work_eb3f1f53df6e5919ae30528c764a27ee_md5.webp deleted file mode 100644 index fef9cd5b2..000000000 Binary files a/topics/design/assets/how-a-design-system-work_eb3f1f53df6e5919ae30528c764a27ee_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_41a4d5306abb2cbebf67b66af3bb7377_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_41a4d5306abb2cbebf67b66af3bb7377_md5.webp deleted file mode 100644 index 76e6b3467..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_41a4d5306abb2cbebf67b66af3bb7377_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_79c08449fbd6f94151675d1e9b64eb98_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_79c08449fbd6f94151675d1e9b64eb98_md5.webp deleted file mode 100644 index e675cdd9d..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_79c08449fbd6f94151675d1e9b64eb98_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_8f1780f4af0d9ab2db8f05e84a8d064a_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_8f1780f4af0d9ab2db8f05e84a8d064a_md5.webp deleted file mode 100644 index f1c70cab1..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_8f1780f4af0d9ab2db8f05e84a8d064a_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_9e4f26e9fc6da86f7941f0da35485279_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_9e4f26e9fc6da86f7941f0da35485279_md5.webp deleted file mode 100644 index 0c0913380..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_9e4f26e9fc6da86f7941f0da35485279_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_a312bb97ae7e137118339c676689e1dc_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_a312bb97ae7e137118339c676689e1dc_md5.webp deleted file mode 100644 index 4375d014c..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_a312bb97ae7e137118339c676689e1dc_md5.webp and /dev/null differ diff --git a/topics/design/assets/six-things-i-extracted-from-design-thinking_d2c2897e585f20d406067bc9aa6d628d_md5.webp b/topics/design/assets/six-things-i-extracted-from-design-thinking_d2c2897e585f20d406067bc9aa6d628d_md5.webp deleted file mode 100644 index ec003b04a..000000000 Binary files a/topics/design/assets/six-things-i-extracted-from-design-thinking_d2c2897e585f20d406067bc9aa6d628d_md5.webp and /dev/null differ diff --git a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-1_cdf0d4d3eccb21431ec5e9c04b336c24_md5.webp b/topics/design/assets/the-principle-of-spacing-in-ui-design-part-1_cdf0d4d3eccb21431ec5e9c04b336c24_md5.webp deleted file mode 100644 index c32189237..000000000 Binary files a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-1_cdf0d4d3eccb21431ec5e9c04b336c24_md5.webp and /dev/null differ diff --git a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_7edc02bcf0ab2561f68d9869b7addbac_md5.webp b/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_7edc02bcf0ab2561f68d9869b7addbac_md5.webp deleted file mode 100644 index 79272d3c1..000000000 Binary files a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_7edc02bcf0ab2561f68d9869b7addbac_md5.webp and /dev/null differ diff --git a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_83a1ac346ab98c836ab637c14c3ac9cf_md5.webp b/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_83a1ac346ab98c836ab637c14c3ac9cf_md5.webp deleted file mode 100644 index 43cd53156..000000000 Binary files a/topics/design/assets/the-principle-of-spacing-in-ui-design-part-2_83a1ac346ab98c836ab637c14c3ac9cf_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_016986df82ac5332b050a9cf55425f3b_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_016986df82ac5332b050a9cf55425f3b_md5.webp deleted file mode 100644 index d8ce85740..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_016986df82ac5332b050a9cf55425f3b_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_1c3f1857f449f57c470ce40efff1bd01_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_1c3f1857f449f57c470ce40efff1bd01_md5.webp deleted file mode 100644 index b9eb8103b..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_1c3f1857f449f57c470ce40efff1bd01_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_9ebb07075efeda26bfeb2a82876bd2ef_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_9ebb07075efeda26bfeb2a82876bd2ef_md5.webp deleted file mode 100644 index 9994023cb..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_9ebb07075efeda26bfeb2a82876bd2ef_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_a46ebf821728ad559bf23e5e6bd7d5a9_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_a46ebf821728ad559bf23e5e6bd7d5a9_md5.webp deleted file mode 100644 index d744f3dfd..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_a46ebf821728ad559bf23e5e6bd7d5a9_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_a70427baf7d24194ab5037713ee0a063_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_a70427baf7d24194ab5037713ee0a063_md5.webp deleted file mode 100644 index 544b1f63a..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_a70427baf7d24194ab5037713ee0a063_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_b37bb057d318c523f8de54660823255a_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_b37bb057d318c523f8de54660823255a_md5.webp deleted file mode 100644 index f82b0ef20..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_b37bb057d318c523f8de54660823255a_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-best-practices-dwarves_f7bcaa80f7503bce54c8f48be4535454_md5.webp b/topics/design/assets/ui-design-best-practices-dwarves_f7bcaa80f7503bce54c8f48be4535454_md5.webp deleted file mode 100644 index 7a5efddda..000000000 Binary files a/topics/design/assets/ui-design-best-practices-dwarves_f7bcaa80f7503bce54c8f48be4535454_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-fundamental_2f86f2fad6448f61bf19fa256a25c973_md5.webp b/topics/design/assets/ui-design-fundamental_2f86f2fad6448f61bf19fa256a25c973_md5.webp deleted file mode 100644 index 63ab153ad..000000000 Binary files a/topics/design/assets/ui-design-fundamental_2f86f2fad6448f61bf19fa256a25c973_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-fundamental_4718c084b78a2e641043dff2ffb94dad_md5.webp b/topics/design/assets/ui-design-fundamental_4718c084b78a2e641043dff2ffb94dad_md5.webp deleted file mode 100644 index 403ebb2d5..000000000 Binary files a/topics/design/assets/ui-design-fundamental_4718c084b78a2e641043dff2ffb94dad_md5.webp and /dev/null differ diff --git a/topics/design/assets/ui-design-fundamental_5c2e41286c6c2f47ccdbe589a50c4033_md5.webp b/topics/design/assets/ui-design-fundamental_5c2e41286c6c2f47ccdbe589a50c4033_md5.webp deleted file mode 100644 index da26922bd..000000000 Binary files a/topics/design/assets/ui-design-fundamental_5c2e41286c6c2f47ccdbe589a50c4033_md5.webp and /dev/null differ diff --git a/topics/design/design-better-mobile-application.md b/topics/design/design-better-mobile-application.md deleted file mode 100644 index 3f833ac3b..000000000 --- a/topics/design/design-better-mobile-application.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: Design better mobile application -description: Explore key differences between iOS and Android operating systems for UX/UI designers, including performance, development, privacy, and app platforms to create better mobile applications. -date: 2020-05-01 -tags: - - design -redirect: - - /vofZhQ ---- - -This workshop wraps up the useful information on 2 types of operating system for UX/UI designers, based on the personal research on iOS and Android. The purpose is to create an application that brings comfortability and a great experience to the end user. - -## Introduction to iOS & Android - -### Adjustment & privacy - -- iOS has better privacy protection since it belongs exclusively to iOS, but this leads to the limitation of function adjustment. -- Android uses an open source code, so it can be adjusted and modified as needed. This is called AOSP (Android Open source Project) - -### Hardware - -- iOS can optimize the hardware and stabilize its platform, and its application has great compatibility with the devices in the ecosystem -- Android: Smartphones of Android usually get lagged during the using process. The home screen may get frozen and cannot respond to the request. - -### Performance - -- iOS manages RAM better, so the performance is fast and barely has any error -- Android: The performance is slow easy to get config that causes errors - -### Application upload on Appstore/ CH Play - -- iOS takes more time to upload: 7 days on average due to personal expert verification -- Android takes less time to upload thanks to automated test - -### Version updates - -- iOS: Update version can be instantly updated for every iOS model, which makes it convenient for every device of the iOS ecosystem (iPhone, iPad, Macbook): convenient in file synchronization, upload photos from iCloud to other devices; Airdrop between iOS & iOS, receive desktop notification for incoming calls, text messenger,… -- Android: Updates on Android usually take more time (due to the diversity of hardware), not to mention some old models can be skipped for version update. This may refer to the strategy of up-sell, people feel the need to purchase the new model once the old one can't be updated. - -### Language - -- iOS: supports over 100 languages (due to the diversified nationality of users) -- Android: 34 languages - -### Development environment - -- iOS: Xcode -- Android: Android studio - -The invention of studio later that supports developer to code on cross platform, therefore the coding process no longer needs to be divided into 2 types of platform. - -### Programmed in - -- iOS: C, C++, Java -- Android: Objective C, C++ - -### Open source - -- iOS: iOS Kernel isn't a source, so it must be based on Darwin OS as open source -- Android: Kerner, UI and some other standard applications - -### Development cost - -- iOS: affordable price -- Android: cost will be double or triple compared to iOS, due to different types of model - -## The platforms to build application - -Based on the programming languages, mobile application can be categorized into 4 types - -- **Native app**: applications written to work on a specific device platform, using the suitable language for each platform, such as Java for Android, Objective C for iOS, C# for Windows Phone. For example, games written for iOS cannot be used in Android. -- **Web-based app**: A computer program that utilizes web browsers and web technology to perform tasks over the Internet. This type of application is run on web platform, written by web languages like HTML5, CSS, Javascript of jQuery Mobile. Basically this is a website with application interface and data is loaded from browser. -- **Hybrid app**: combines the elements of both native and Web applications. The basic parts of the application are still written in web language, but it will be placed on the native container. -- **Cross platform** **(Multi-platform)**: one codebase is applied for all platforms. Developers only need to write the codebase once, then translate or transpile it into different Native app versions that for each different platform. This is known as the most cost-optimized one. - -Still, there are some researches claim that Hybrid app and Cross platform are the same. Some said these two are different, so I'll put these into 4 types, so the traits of each type can be described clearer. - -![](assets/design-better-mobile-application_56fa2f8d701c84b46a715ca629e77f27_md5.webp) - -### Pros & cons of each platform - -![](assets/design-better-mobile-application_7c641059d4d0fad9e7fd2c85c45edbaa_md5.webp) - -![](assets/design-better-mobile-application_3db029547cf3f0df87733d42e499f2b5_md5.webp) - -## Useful tips to create effective mobile application - -### Read the human interface guidelines - -**Android:** - -- [Google design](https://design.google/resources/) -- [Material design](https://material.io/design/introduction#goals) - -**Window:** [Universal Windows Platform Apps](https://docs.microsoft.com/en-us/windows/win32/uxguide/how-to-design-desktop-ux) - -**iOS:** [iOS Human Interface Guidelines](https://developer.apple.com/design/human-interface-guidelines/ios/overview/themes/) - -### Talk to developers - -- Understand the difficulties -- Collect insight -- Flexibility and effectiveness in App design - -### Component structure between Android & iOS - -- Minimum Tap Target Size -- Main App Navigation -- Primary Navigation Destination -- Secondary Navigation Destination -- Primary Button/ Action -- Secondary Action -- Selection Control -- 'Undo' Pattern on iOS & Android -- App Icon Size -- Top-of-Screen Navigation -- Back Pattern -- Search Bar -- Action Menus -- Date Picker -- Tabs diff --git a/topics/design/design-resourcestools.md b/topics/design/design-resourcestools.md deleted file mode 100644 index eb3c5f1f2..000000000 --- a/topics/design/design-resourcestools.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: Design resourcestools -description: Discover top UX design resources, including expert articles, communities, podcasts, books, and tools to improve user experience and master UX design techniques effectively. -date: 2018-03-25 -tags: - - design -redirect: - - /L4iAwg ---- - -Besides learning the UI design, the most important thing is to learn UX design to create a good user experience. Helps the user get the goal when in your app. - -This compilation is synthetic all the websites or tools to help you improve your knowledge of UX Design. All of these websites where UX designers around the world often go to learn, just as you usually do on Behance or Dribbble to find trends, styles or ideas. - -## Information and guideline - -### Publications - -- **[Nielsen Norman Group](https://www.nngroup.com/articles/)** - Original research from one of the most respected authorities in UX. -- **[Luke W](https://www.lukew.com/ff/)** - Very short, straightforward, and visual articles from a Product Design veteran. -- **[Smashing Magazine](https://www.smashingmagazine.com/)** - High quality, vetted design content from a collection of experts. -- **[UX Magazine](https://uxmag.com/)** - A large collection of articles covering virtually every UX subset. -- **[UX Matters](https://www.uxmatters.com/)** - Long form, evergreen articles dating back to before UX was a thing. -- **[Usability Geek](https://usabilitygeek.com/)** - Wide range of usability and tech related articles and guides. -- **[UX Mastery](https://uxmastery.com/)** - A giant guide covering all of the basics for getting started in UX. -- **[52 Weeks of UX](http://52weeksofux.com/tagged/week_1)** - Thought-provoking niche UX topics from Josh Porter and Josh Brewer. -- **[InVision Blog](https://www.invisionapp.com/blog)** - High quality design articles, guides, and interviews. - -## Communities - -- **[Designer News](https://www.designernews.co/)** - Board for design discussion, critique, and content sharing. - -- **Designer hangout** - Slack community (more than 6,000 members) that is focused on UX - -- **[/r/UserExperience](https://www.reddit.com/r/userexperience/)** - Subreddit for UX related content, tools, and conversation - -### Podcasts - -- **[UX and Growth Podcas](https://austinknight.com/podcast/)** - A tactical, actionable UX podcast that I co-host with two friends. -- **[User Defenders](https://userdefenders.com/)** - In-depth interviews with influential UX designers, hosted by Jason Ogle. -- **[Design Review](http://www.designreviewpodcast.com/)** - Weekly reviews of products and their user experiences. -- **[What Is Wrong with UX?](https://itunes.apple.com/us/podcast/what-is-wrong-ux-users-know/id980133198?mt=2)** - Laura Klein and Kate Rutter discuss current issues in UX. -- **[Design Details](https://spec.fm/podcasts/design-details)** - Casual interviews with designers and product creators. -- **[Dorm Room Tycoon](https://drt.fm/)** - Interviews with major influencers in design, business, and technology. - -### Books - -![](assets/design-resourcestools_1d108bd8a65ff50110c20d550f63945e_md5.webp) - -A foundational book on UX and Design, introducing design as a competitive advantage in business. - -![](assets/design-resourcestools_9863db6fd97f581087152cab779966e2_md5.webp) - -A straightforward book on usability, serving as a great introduction to UX. - -![](assets/design-resourcestools_35a07daafda041800eb792f9dfb52744_md5.webp) - -An introduction to statistics and data visualization (and how they can both be manipulated). - -![](assets/design-resourcestools_22750df99a8ae9d95127999f8d41614b_md5.webp) - -Part of the Lean Startup movement, this book introduces the process for Lean UX. - -## Tool and resources - -### Quantitative - -- **[Google Analytics](https://www.google.com/analytics/#?modal_active=none)** - Free, standard yet robust web analytics with wide adoption. -- **[HubSpot](https://www.hubspot.com/)** - In-depth analytics, A/B testing, SEO, CRO, and more. -- **[Mix panel](https://mixpanel.com/)** - Free, fully custom and integrated mobile & web analytics platform. -- **[Heap](https://heapanalytics.com/?referral=5djmk)** - Automated analytics platform that captures every action. -- **[Crazy Egg](https://www.crazyegg.com/)** - Heat mapping, click mapping, scroll mapping, and more. -- **[KISS metrics](https://www.kissmetrics.com/)** - Easily configured funnels and reports based on user behavior. -- **[Optimizely](https://www.optimizely.com/)** - Javascript-based A/B testing and multivariate testing platform. - -- **[Google Optimize](https://optimize.google.com/optimize/home/#/accounts/2740283315/containers/8631801)**- Free A/B testing platform that’s integrated with Google Analytics. - -### Testing - -- **[UserTesting.com](http://usertesting.com/)** - Fully integrated user testing platform with a participant panel. -- **[UserZoom](https://www.userzoom.com/)** - All-in-one qualitative research and user testing platform. -- **[Try](https://www.trymyui.com/)**\*\*\***\*[My](https://www.trymyui.com/)**\*\*\*\*\*\*[UI](https://www.trymyui.com/)\*\* - Remote unmoderated usability testing panel and platform. -- **[Verify App](https://verifyapp.com/)** - Quickly test where users would click in wireframes and mockups. -- **[Loop11](https://www.loop11.com/)** - Platform for testing wireframes and prototypes, usability, and accessibility. - -### Diagrams - -- **[Gliffy](https://www.gliffy.com/)** - Free Chrome plugin for drawing diagrams, flowcharts, and more. -- **[Google Drawings](https://docs.google.com/drawings/d/1saUBNhMvr-v-nunOZ73Pp-rqrPp-r4EeT2Ml2p2lcng/edit)** -Create, share, and collaborate on drawings in your Google Drive. - -- **[Coggle.it](http://coggle.it/)** - Visually map out concepts during brainstorming sessions. - -- **[XMind](http://www.xmind.net/)** - Free tool for quick and easy to edit Mind Mapping. - -- **[Keynote](https://www.apple.com/keynote/)** - Great for creating personas, task and competitive analyses, etc. - -### Wireframes - -- **[Balsamiq](https://balsamiq.com/)** - Basic, straightforward wireframe tool with web and desktop versions. diff --git a/topics/design/design-system.md b/topics/design/design-system.md deleted file mode 100644 index 316fcd0c9..000000000 --- a/topics/design/design-system.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -title: Design system -description: Learn how to create a design system that unites teams, speeds up design workflows, and improves product quality with reusable components, visual language, design tokens, and UI libraries. -date: 2018-03-23 -tags: - - design -redirect: - - /O4JEzg ---- - -Now having a Design system in place unites product teams around a common visual language, accelerates your design process ten-fold and gives you a strong foundation to build any project. Learn how you can create a design system and help your team improve product quality when increase the speed of your workflow. - -## Introduction - -Today there are many the companies products compete with each other, the company should have one of the conditions: - -- Make good and famous products -- Design fast and quality -- Have experience consultant and solve problems for customers - -So apart from those two conditions, design systems enable a team to increase products faster by making design reusable makes scale possible. A design system is a collection of reusable components, guided by clear standards, that can be assembled together to build any number of applications. - -When you first start working with a new system you may find yourself slightly hindered, because it is a different way of approaching a project, and something a little different than you’ve maybe become accustomed to. - -But honestly, if you devote one's time to create this Design system, you'll find yourself working on auto-pilot. You can drive all projects and it will seem like your child. - -## Design - -Starting a design system can feel daunting. There are so many things to consider: the design style, how to design for modularity and scalability, how it will be used by other teams, how to sell the idea to the decision makers in the company. Where is a designer to start? - -### 1.Who should be involved - -Before beginning work on your design system, take a moment to think about the team you’ll need to bring it to life. Who needs to be involved? Spoiler alert! You’re going to need more than just designers. - -Here’s a quick list of the disciplines that can be represented in your team to create an effective design system: - -- **Designers** to define the visual elements of the system -- **Front-end developers** to create modular, efficient code -- **Content strategists** who can help the team nail the voice and tone of the system -- **Researchers** who can help you understand customer needs -- **Performance experts** who can ensure your system loads quickly on all devices -- **Product managers** to ensure the system is aligned with customer needs -- **Leaders** (VPs and directors) to champion and align the vision throughout the company, including up to executive leadership - -### 2. Choosing the right team model - -- **The solitary model:** an “overlord” rules the design system. - -![](assets/design-system_cfe9acf6fd2d1adde475eac22289689b_md5.webp) - -The team model that brings people together is as important as the team creating the design system. In “Team Models for Scaling a Design system,” design systems veteran Nathan Curtis outlines 3 popular team models used by many companies. - -- **The centralized team model:** a single team maintains the design system as their full-time job. - -![](assets/design-system_29a2cbbe18c551fb6c02d550c7eba526_md5.webp) - -- **The federated model:** team members from across the company come together to work on the system. - -![](assets/design-system_777f120275d29fe97d756a8d98d5be91_md5.webp) - -- Other model - -A hybrid design system team model that we used at Salesforce—a central team and members of other teams come together to manage and govern the system. - -![](assets/design-system_b187e07f45c64d9eabd7e524b39a31ed_md5.webp) - -### 3. Interviewing customers - -Like any product in a design process, it’s important to do your research. Who will be using your design system and how will they use it? Answer this question is the customer. [You should difference between the Customer and the User](https://www.uxpin.com/studio/blog/customer-experience-vs-user-experience-why-the-difference-matters/). The customer will use your design system. So your design system will get used much more often if you create it to fit into the workflow of other teams. By interviewing customers, you can pinpoint problems ahead of time, define principles that will help others use the system properly, and focus your energies on the most important things. - -This process can include: - -- **Interviews** of key (potential) contributors, influencers and leaders to assess perspective, attitudes, culture, and existing practices. -- **Survey**ing a broader organization of stakeholders attitudes and posture towards a system, priorities/needs, aspirations, and threats. -- **Requirements** gathering via task analysis, tech planning, and convention setting (using tools like [Brad Frost](https://twitter.com/brad_frost)’s [Front End Questionnaire](https://github.com/bradfrost/frontend-guidelines-questionnaire)). -- **Product tours** to immerse in as-is products and in-flight designs to which the system will apply, taking screenshots and notes. -- **System(s) reviews** assessing as-is design assets, code libraries, standards documentation depth and quality, and governance models. - -### 4. Creating a visual inventory - -With insights in hand from customer interviews, it’s time to take an inventory. There are 2 types of interface inventories to be created: - -- An inventory of the visual attributes (such as spacing, color, and typography), which will help create a codified visual language -- An inventory of each UI element (such as buttons, cards, and modals), which will help create a UI library of components - -Creating the visual identity isn’t something that will be created overnight. It takes time. Sometimes it’s as clear as day as to what is needed, other times it takes time for the building blocks to fall into place. Once in place, it’s important that the fundamentals are captured and documented at a high level. The likes of use of color, typography and style of iconography is key to creating consistency across a platform. - -As we start to take inventory, it’s good practice to take a look at the CSS used to create all of those elements you just captured in your visual inventory. Use a tool like **[CSS Stats](http://cssstats.com/)** to see how many rules, selectors, declarations, and properties you have in your style sheets. More relevant, it will show you how many unique colors, font sizes, and font families you have. It also shows a bar chart for the number of spacing and sizing values. This is a great way to see where you can merge or remove values. - -![](assets/design-system_a9cdca95e3ffef0e55cd90fdec84527a_md5.webp) - -**Do a UI inventory audit** -Before you start anything, its best to identify how inconsistent the current build is. This works in two ways. It helps identify the reason as to why you’re doing it, to identify how inconsistent everything is but it should help you get the backing of the business as to why exactly you’re creating the design system; to create consistency across the platform. - -- **Colors:** What is the color palette used on the platform? Explain how, where and why we use certain colors. -- **Typography:** What typeface is used on the platform? Summarizes rules around weighting, sizing, vertical alignment etc? -- **Iconography:** What is the generic style for icons? It will explain the rational as to why we have specific styles for different icon families. -- **Grid/Layouts:** What grid system is used across the platform? Explain the use of the grid and the high level idealism of our layouts. -- **Interactions:** What do people expect to see when they interact with our site? Give an overview of our standard interactions. -- **Animations:** How do we approach animations? Explain the reason for animations on the platform and our constraints around using them. -- **Design resources**: A central point for assets to be easily downloaded for external partners. Color swatches, logo’s, icon sets etc. - -An example of a UI Audit: Brad Frost has put together a great article around how you go about doing a UI audit. - -![](assets/design-system_3e3694f6596e6a67a2a10b4bb6ea8eed_md5.webp) - -Link source: [http://bradfrost.com/blog/post/interface-inventory/](http://bradfrost.com/blog/post/interface-inventory/) - -### 5. Creating a visual design language - -If we break apart each component of a design system we find that these fundamental elements make up its visual design language: - -- Colors -- Typography (size, leading, typefaces, and so on) -- Spacing (margins, paddings, positioning coordinates, border spacing) -- Images (icons, illustrations) - -Depending on your needs, you may also include the following to further standardize the user experience: - -- Visual form (depth, elevation, shadows, rounded corners, texture) -- Motion -- Sound - -### 6. Design token - -Before we dive into visual design standards, you should discuss design tokens. Design tokens are the “subatomic” foundation of a design system implementation. At its simplest, they’re name and value pairs stored as data to abstract the design properties you want to manage. With the values for all design tokens stored in a single place, it’s easier to achieve consistency while reducing the burden of managing your design system. - -Example of design token: [https://www.lightningdesignsystem.com/design-tokens/](https://www.lightningdesignsystem.com/design-tokens/) - -![](assets/design-system_0018d0d34bb133667d90e34ce27c5fd0_md5.webp) - -[https://uxdesign.cc/design-tokens-for-dummies-8acebf010d71](https://uxdesign.cc/design-tokens-for-dummies-8acebf010d71) - -The workflow of design tokens would look like: - -![](assets/design-system_d4b5482e66a2d78fe46a315c5d2a4646_md5.webp) - -1. The designer would update the color in the design tool. -2. The design tool updates design tokens files according to targeted platform. -3. Developers only have to retrieve or “pull” updated files and use it in their project. - -Currently, the only way to create a design tokens is by using [Theo](https://github.com/salesforce-ux/theo), it made by Salesforce. What Theo does is simple: it takes as input a tech agnostic file format like JSON or YAML and outputs tech specific files for each platforms. - -### 7. UI Library (Pattern Library) - -After you complete the inventory, you can merge and remove what you don’t need (either in a spreadsheet or even directly in a code refactor if you want more immediate change). Also, document what the component is and when to use it. This will become your UI library (or pattern library, or component library, depending on what your organization chooses to call it.). - -![](assets/design-system_b0d28b9d30e9d838276303ad69f6e89a_md5.webp) - -Most design system documentation includes the component’s name, description, example, and code. Others may show meta data, release histories, examples, and more. What matters most is that you show what’s necessary for your team to get your work done. - -Process: [https://medium.com/@jgunnison/pattern-library-workflow-ba9cc486159e](https://medium.com/@jgunnison/pattern-library-workflow-ba9cc486159e) - -### 8. Style guide - -[https://medium.muz.li/how-to-create-a-style-guide-from-scratch-tips-and-tricks-e00f25b423bf](https://medium.muz.li/how-to-create-a-style-guide-from-scratch-tips-and-tricks-e00f25b423bf) - -After you understand this process of the design system, this guide will help you create a UI library in Sketch. -Let’s take a step-by-step below here how you can create your design system. diff --git a/topics/design/design-tips-tricks.md b/topics/design/design-tips-tricks.md deleted file mode 100644 index c28a02427..000000000 --- a/topics/design/design-tips-tricks.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Design tips tricks -description: Discover key UX design principles to create user-friendly forms, avoid common product design mistakes, and simplify online checkout processes for better user experience and higher conversion rates. -date: 2018-03-24 -tags: - - design -redirect: - - /AiWsRQ ---- - -- [https://uxplanet.org/the-18-must-do-principles-in-the-form-design-fe89d0127c92](https://uxplanet.org/the-18-must-do-principles-in-the-form-design-fe89d0127c92) -- [https://blog.prototypr.io/4-common-mistakes-ux-designers-should-avoid-during-product-design-process-cace92d8b8b3](https://blog.prototypr.io/4-common-mistakes-ux-designers-should-avoid-during-product-design-process-cace92d8b8b3) -- [https://uxplanet.org/why-is-online-checkout-so-difficult-and-how-to-simplify-it-eeb3da2ea530](https://uxplanet.org/why-is-online-checkout-so-difficult-and-how-to-simplify-it-eeb3da2ea530) diff --git a/topics/design/design-token.md b/topics/design/design-token.md deleted file mode 100644 index 821e0451b..000000000 --- a/topics/design/design-token.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Design tokens in the AI era -description: Design tokens create consistency across platforms by centralizing visual attributes. AI tools now automate token creation and management, making design systems accessible to non-designers. -date: 2025-06-08 -authors: - - tieubao - - claude -tags: - - design-tokens - - design-systems - - ai-tools - - automation ---- - -**Design tokens are the shared language between designers and developers.** These platform-agnostic variables store visual attributes like colors, typography, and spacing, ensuring consistency across web, mobile, and other platforms. AI tools now automate much of the heavy lifting in token creation and management. - -## What design tokens accomplish - -Design tokens fall into two categories: global tokens (`blue.default` for a color) and semantic tokens (`button-primary` for context-specific use). They centralize design decisions in JSON files or similar formats, eliminating hard-coded values throughout codebases. - -This approach enables rapid updates across platforms. Change one token value and every instance updates automatically. Teams can implement theming systems, like light and dark modes, by swapping token sets. The result is fewer errors, faster development, and unified visual consistency. - -![](assets/design-token.png) - -## Why AI transforms token workflows - -AI tools analyze design files from Figma or Sketch, automatically generating tokens for colors, fonts, and spacing. This eliminates hours of manual extraction work. AI suggests semantic token names based on usage patterns, creating intuitive labels like `text-secondary` for gray caption text. - -Modern AI systems also handle distribution. An AI agent can detect token changes in design files, update code repositories, and trigger deployment pipelines automatically. This ensures real-time consistency across all platforms without manual coordination. - -## How to implement AI-powered tokens - -Start with tools like Style Dictionary for token management or Figma plugins with AI integrations. These platforms streamline initial setup and ongoing maintenance. For non-designers, AI-powered natural language interfaces allow updates through simple commands like "make buttons darker." - -However, maintain human oversight. AI can suggest accessibility improvements for WCAG compliance or generate dynamic themes, but teams should validate outputs against brand guidelines and user needs. The goal is leveraging AI efficiency while preserving creative control and design quality. - -AI makes design tokens more accessible, but the fundamental value remains human-centered design decisions executed consistently at scale. diff --git a/topics/design/design-workflow.md b/topics/design/design-workflow.md deleted file mode 100644 index 4900ce1d1..000000000 --- a/topics/design/design-workflow.md +++ /dev/null @@ -1,150 +0,0 @@ ---- -title: Design workflow -description: Learn how to improve product design with user research, job stories, guerrilla usability testing, wireframing, UI principles, prototyping, and design systems for consistent and effective user interfaces. -date: 2018-03-22 -tags: - - design -redirect: - - /5urfoA ---- - -## Project research - -The goal of this step is to figure what the problems you need to solve using your design from asking the people involved such as our customers (businesses owners) and the users, the people who will use your design outcome directly. We don't guess or assume things. - -There are 2 types of research you can do to figure this out: - -- **Quantitative:** These are the things we can measure. Examples include analytics that communicate customer behavioral patterns and aggregate stats about customer cohorts. -- **Qualitative:** These are things that tell us about the qualities of a product or experience. Customer interviews, for example, give us insights about how a customer feels, which can provide a lot of insight into what motivates their behavior. - -You can create a provisional persona of potential user base on your research and people around you who you think they are able to your object of study. - -![](assets/design-workflow_aad2cd76d88ab384bd09ba16793dab87_md5.webp) - -## Job story & scenario - -After collecting information and figuring out the above expected goals, we continue to conduct Job Stories and come up with Wire-frames. - -### Writing job stories - -This is an example of a Job Story. - -"When I find a good movie, I want to share that movie for others, so that people could know that I have a good taste of movies." - -After that, you write a scenario base on your job story. - -When you write a Job Story, the story need three things: **_Situation, Motivation _**and**_ Outcome_**. - -Your final design should be a list of jobs to be done which help the users achieve their goals. Therefore, writing down job stories will help you know exactly what you should put in your final design for your users. - -## Guerrilla usability testing - -Base on your job story and the scenario, you will conduct guerrilla usability to test you hypothesis. - -Before you ask them to do tasks, you should ask them to imagine they are in your situation and what they need. - -Example of questions: - -1. If you are looking for a place where you’d like to travel, how would you do that? -2. Let’s say you found a place you like, what do you do to check out that place? - -After that, you should make statistical tables base on your test. - -3. You’re interested in the place, and you decide to plan on travel there, what would you do next? - -## Identifying and prioritizing pain points - -After you review the user interviews, you write each users' pain points onto Post-Its. Then you categorize pain points with Affinity Mapping and prioritize them with 2x2 Matrix. - -![](assets/design-workflow_d8fe23347d8bff5a0be6ae56bd0fc438_md5.webp) - -## Flowchart - -Next step, you draw a flowchart what is a diagram of the sequence of movements or actions of people or things involved in a complex system or activity. - -![](assets/design-workflow_1ce44520b17bffd06b033406e0d794a6_md5.webp) - -## Design decisions - -**Sketch wire-frames\*\*** - -![](assets/design-workflow_d6a68d2878bdf01a430afc77624683a2_md5.webp) - -Sketching wire-frames using pencil & paper make easier to iterate your design and get feedback. - -**Here are \*\***[some reasons](https://www.designbetter.co/principles-of-product-design/pencils-before-pixels)\***\* why:** - -1. _Pencils are inclusive_. They’re not just for designers—anyone can use a pencil to express their ideas clearly. The pencil is the great equalizer. -2. _Pencils are low-fi_. Quick sketches give no impression of a complete thought, signaling to all that it’s okay to offer feedback. -3. _Pencils aren’t fiddly_. Instead of getting lost in software settings or style, you’ll focus on your *ideas*. -4. _Pencils are fast_. You can explore vastly different solutions to the same problem in minutes, and you won’t feel bad throwing your sketches out because you invested so little time. - -But you can also choose your right tools for better delivery, such as [Balsamiq](https://balsamiq.com/index.html) - -## UI design - -We believe you already had a good taste of design so this step should mostly based on your creative. But it's still good to get these principles as we believe **grid, visual hierarchy, typography, icons** and **colors** are the things that make useful, attractive user-interfaces. - -Getting to know how we humans read with digital screens and interact with computers is also helpful to build engaged users-interfaces. - -**Principles** - -[https://www.smashingmagazine.com/2018/02/comprehensive-guide-ui-design/](https://www.smashingmagazine.com/2018/02/comprehensive-guide-ui-design/) - -[https://medium.com/hh-design/crash-course-ui-design-25d13ff60962](https://medium.com/hh-design/crash-course-ui-design-25d13ff60962) - -[https://polaris.shopify.com/visuals](https://polaris.shopify.com/visuals) - -**Use of colors** - -[https://medium.com/@erikdkennedy/color-in-ui-design-a-practical-framework-e18cacd97f9e](https://medium.com/@erikdkennedy/color-in-ui-design-a-practical-framework-e18cacd97f9e) - -[https://uxplanet.org/the-most-important-color-in-ui-design-d4f23aefffdf](https://uxplanet.org/the-most-important-color-in-ui-design-d4f23aefffdf) - -**Typography for the digital screens** - -[https://www.smashingmagazine.com/2014/09/balancing-line-length-font-size-responsive-web-design/](https://www.smashingmagazine.com/2014/09/balancing-line-length-font-size-responsive-web-design/) - -**Icons** - -[https://medium.com/@tubikstudio/icons-in-ui-design-great-power-of-small-details-7942df655a04](https://medium.com/@tubikstudio/icons-in-ui-design-great-power-of-small-details-7942df655a04) - -## Delivery - -Getting better at documentation and presentation is also required as a designer. - -**Writing** - -Many times you will find your self forgetting why you put that icon into that place, why you did choose your colors, etc. Writing down is helpful for your brain since you don't have to remember so many stuffs, it's also helpful for others when they want to understand your intends without bugging you through emails while you are on vacation. - -**Prototyping** - -Getting to use your design is exciting. Waiting for it to be approved and built up from scratch through many debates with developers is not. So give yourself & them a [working prototype](https://marvelapp.com/54hd8ia/screen/25676944). Use [marvelapp](https://marvelapp.com/). - -**Design System** - -Consistent is a key to good design. It might be easier for you to keep your design consistent throughout your design files. But it is not the same to developers. Building a design system or a design guideline for your design is healthy for your design and helpful for your fellow developers to keep your design's consistency on the real production app. - -Delivering your design with a design system is a **MUST.**So keep that in mind in day one on designing it. - -- **Read more** on how to create a design system based on your design using tools like Sketch or Figma. - -[https://www.sketchapp.com/docs/libraries/](https://www.sketchapp.com/docs/libraries/) - -[https://blog.figma.com/components-in-figma-e7e80fcf6fd2](https://blog.figma.com/components-in-figma-e7e80fcf6fd2) - -[https://blog.figma.com/team-libraries-in-figma-409fa5e20f7](https://blog.figma.com/team-libraries-in-figma-409fa5e20f7) - -[https://blog.figma.com/team-library-1-0-d1427092323a](https://blog.figma.com/team-library-1-0-d1427092323a) - -**Guideline** - -This step will support the development team to understand what system you are doing so that they can follow up faster. - -![](assets/design-workflow_a19657465a449ff547f2c9008bbfab8c_md5.webp) - -## Review, feedback - -1. Use tools like Marvel, Figma to present your design. Marvel should be used when you want to present the interactive prototypes, while Figma is better to present general designs, just import your final design here to present & collect feedback. Don't forget to let people know by shooting a link via Slack/Hygger. -2. Feedback shouldn't be taken personally. -3. Feedback should be constructive given with reasons that aligned with listed goals of the product. diff --git a/topics/design/designing-for-forgiveness.md b/topics/design/designing-for-forgiveness.md deleted file mode 100644 index e4e2ed260..000000000 --- a/topics/design/designing-for-forgiveness.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: "Designing for forgiveness: creating error-tolerant interfaces" -description: In this article, we will look at some simple principles for designing error-tolerant interfaces and how to apply them to improve user experience. -date: 2024-08-23 -authors: - - Maniub102 -tags: - - design - - ux -redirect: - - /J1QZUA ---- - -In the world of digital products, ensuring that users have a smooth experience is crucial. One key part of this is designing interfaces that can handle user mistakes. This concept, known as "designing for forgiveness," means creating systems that not only help users avoid errors but also assist them in recovering quickly when they make mistakes. In this article, we will look at some simple principles for designing error-tolerant interfaces and how to apply them to improve user experience. - -### Understanding the problem: user errors are inevitable - -No matter how well we design a system, users will still make mistakes. These mistakes can be as simple as clicking the wrong button or as complex as misunderstanding how a feature works. Understanding that errors will happen is the first step in creating interfaces that can handle them effectively. - -**Common Types of Errors**: - -1. **Accidental errors**: These happen when users unintentionally do something wrong, like clicking the wrong button or entering the wrong information. -2. **Errors due to lack of understanding**: Sometimes, users don’t fully understand how a feature works or they are new to the interface. -3. **Errors due to unclear design**: If the interface is not easy to understand, it can lead to confusion and mistakes. - -Given these types of errors, the next question is: How can we design interfaces that not only help reduce these mistakes but also help users recover easily when they do happen? - -### Provide early warnings - -One way to help reduce user errors is by giving early warnings before they perform an action that could lead to a mistake. This helps users think again before they make a decision or correct the mistake before it causes a problem. - -**Examples**: - -- **Microsoft Word**: When users try to close a document without saving it, Word asks if they want to save their changes before closing. This helps prevent losing important work. - - ![](assets/designing-for-forgiveness-microsoft.png) - -- **Shopify**: As soon as we change an information in the box, a warning of unsaved changes appears right on the top bar. Make sure we must act Discard or Save before leaving. - - ![](assets/designing-for-forgiveness-shopify.png) - -**Benefits**: Early warnings help reduce serious mistakes, protect important data, and make users feel safer when using the system. - -### Design clear and consistent interfaces - -Having a clear and consistent design helps users navigate the system easily and reduces the chance of mistakes. If the design is predictable, users can focus on their tasks without worrying about how the system works. - -**Examples**: - -- **Microsoft Office**: The familiar layout and consistent icons across different versions of Word and Excel help users easily switch between updates without needing to learn the interface all over again. - - ![](assets/designing-for-forgiveness-excel.png) - -**Benefits**: A clear and consistent interface leads to fewer mistakes caused by confusion, improving the overall user experience. - -### Graceful error recovery - -Even with the best design, errors will still happen. That’s why it’s important to have tools that help users recover quickly from their mistakes. - -**Examples**: - -- **Drive**: The “Undo Remove” feature lets users recall an item they’ve just removed, allowing them to correct a mistake before it becomes permanent. - - ![](assets/designing-for-forgiveness-drive.png) - -**Benefits**: These recovery tools reduce frustration and stress, allowing users to continue their tasks without too much disruption. - -### Error prevention through visual feedback - -Giving users real-time feedback about what’s happening in the system helps prevent mistakes before they happen. If users know right away that something is wrong, they can fix it immediately. - -**Examples**: - -- **Twitter**: When users go over the character limit, Twitter highlights the extra text and disables the tweet button. This instant feedback helps users correct the mistake without guessing. - - ![](assets/designing-for-forgiveness-twitter.png) - -- **Discord**: The character limit displayed, and if including the extra text and disable the Save Changes button. - - ![](assets/designing-for-forgiveness-discord.png) - -**Benefits**: Real-time feedback ensures that users can fix mistakes quickly, leading to a more efficient and error-free experience. - -### Provide guidance and support - -To help users avoid mistakes, it’s important to provide clear guidance throughout their experience with the system. - -**Examples**: - -- **Word**: For icons that are not clearly visible, tooltips are the best solution, helping users understand what they are for. - - ![](assets/designing-for-forgiveness-excel-setting.png) - -- **Onboarding in Webflow**: Webflow uses interactive tutorials and visual cues to guide new users through key features, such as adding elements to a webpage. These prompts help users quickly learn how to use the tool effectively, reducing the likelihood of errors - - ![](assets/designing-for-forgiveness-webflow.png) - -**Benefits**: Providing guidance and support from the start helps reduce the number of mistakes, especially for new users, and ensures a better experience with the system. - -### Conclusion - -Designing error-tolerant interfaces is not just about preventing mistakes, but also about helping users recover when they do make mistakes. By applying principles like giving early warnings, clarifying choices, ensuring a consistent design, offering recovery tools, providing real-time feedback, and guiding users, we can create interfaces that make users feel confident and satisfied with the product. - -Incorporating these principles into your design process will lead to more user-friendly products that help users achieve their goals with fewer obstacles and greater satisfaction. diff --git a/topics/design/domain-glossary.md b/topics/design/domain-glossary.md deleted file mode 100644 index 59f1c1eb9..000000000 --- a/topics/design/domain-glossary.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Domain glossary -description: Discover how Retail 2.0 enhances shopping with AR, VR, AI, and personalized experiences, while open banking uses APIs to securely share financial data and transform the banking industry. -date: 2020-04-19 -tags: - - design -redirect: - - /52jn3g ---- - -## T1: Retail 2.0 - -_Definition_ - -- Enhancing consumer engagement within brick and mortar - online/using mobile devices, moving beyond simply blending in the online and the mobile experience, all driven at the heart by sophisticated big data tools that truly enable contextual, personalized and real-time conversations between brands, retailers and consumers. [1] -- Made up of AR (augmented reality), VR (virtual reality), in-store apps, new payment methods and artificial intelligence. - -### Personalized exp - -Designing or producing services and products to meet customer's individual requirements. You’re most likely already familiar with marketing personalization - -### Simplified authentication - -Authentication can be one of the most significant sources of friction for your application’s users. This usability barrier can impact adoption of your product, increase the burden on support operations or product administrators and, in some cases, degrade security. - -### Retail 3.0 - -Haravan: Support offline shop on online selling to trigger O2O (standardize data structure) ––> Create the foundation for big data and 4.0 - -### Retail 4.0 - -![](assets/domain-glossary_9c8390ea196dd16d4d3dd5a56311ba94_md5.webp) - -### O2O - Online to offline - -- Originally, implies an online trigger which prompts the customer to go to a physical location to complete their purchase -- Evolvingly, it is the ability to pay online and then pick up a product in an offline place, such as the retailers' physical store or 3rd party locations. -- Another O2O feature is returning items purchased online to the retailers' offline location.Open banking is also known as "open bank data." Open banking is a banking practice that provides third-party financial service providers open access to consumer banking, transaction, and other financial data from banks and non-bank financial institutions through the use of application programming interfaces (APIs). Open banking will allow the networking of accounts and data across institutions for use by consumers, financial institutions, and third-party service providers. Open banking is becoming a major source of innovation that is poised to reshape the banking industry. - -![](assets/domain-glossary_4bc9024bc646f7b46a1675cd62ff9350_md5.webp) - -```javascript -Takeway -- Open banking is the system of allowing access and control of consumer banking and financial accounts through third-party applications. -- Open banking has the potential to reshape the competitive landscape and consumer experience of the banking industry. -- Open banking raises the potential for both promising gains and grave risks to consumers as more of their data is shared more widely. -``` - -## T2: Open banking - -Open banking is also known as "open bank data." Open banking is a banking practice that provides third-party financial service providers open access to consumer banking, transaction, and other financial data from banks and non-bank financial institutions through the use of application programming interfaces (APIs). Open banking will allow the networking of accounts and data across institutions for use by consumers, financial institutions, and third-party service providers. -Open banking is becoming a major source of innovation that is poised to reshape the banking industry. diff --git a/topics/design/focus-personas-on-tasks-goals-and-motivations.md b/topics/design/focus-personas-on-tasks-goals-and-motivations.md deleted file mode 100644 index e09c9a21f..000000000 --- a/topics/design/focus-personas-on-tasks-goals-and-motivations.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Focus personas on tasks goals and motivations -description: Discover how effective user personas emerge from qualitative research and sensemaking to capture real user experiences, guiding product design with authentic insights rather than assumptions. -date: null -redirect: - - /6MsyosY ---- - -Personas shouldn't be created so much as discovered. First and foremost, personas document research. Personas articulate patterns of thought and feeling. These are attributes that can't easily be quantified. Effective user personas should be an outcome of qualitative research. Personas should be the result of [[≈ Sensemaking | sensemaking]], not measurement. - -This is [[¶ Abductive reasoning | abductive research]]: it doesn't start with a supposition or hypothesis, but begins instead with an openness to evidence that can be assembled into a theory. Persons require attention to what Christian Madsbjerg calls "thick data"—the rich context below the surface that details what an experience feels like rather than simply noting that it occurs. Personas should describe not just what kinds of people use the product, but help teams understand how they understand their relationship to the product and the problem space it occupies. - -Due to a small sample size, it is possible that these qualitative personas will not completely capture a representative portrait of the use base. Some organizations therefore build on qualitative personas with another layer of quantitative research to help validate qualitative findings and identify statistical patterns among the users. This adds a veneer of scientism to personas and provides assurance that they are representative of the user base. However, the creation of statistical personas is far more time consuming and may yield only incremental benefits for the extra work.5 - -Teams and organizations will sometimes create personas based on their best current understanding of the user; these should be clearly described as _proto-personas_ and be _treated carefully as assumptions_ that should be subject to testing. Teams should be reluctant to base any significant decisions on proto-personas unless they're supplemented by more rigorous research. Nevertheless, when research isn't possible, proto-personas may be better than nothing. In the very least, they may serve as reminders that, at the end of the day, work needs to serve the customer. They may also be more useful in Lean UX environments where work will be continually tested and iterated upon even without initial discovery research. - -Alan Cooper emphasizes the importance of selecting a primary persona—the user archetype whose needs must be met at all costs. Most projects will involve multiple personas, but one should be identified as most important. Cooper also suggests that there may be value in creating anti-personas—in other words, defining user for whom you explicitly won't design. - -Personas should be living, breathing documents. As knowledge about the user or customer evolves, so should the personas. - ---- - -#### Related - -- [[Personas communicate research or they communicate assumptions]] - -#### Citations - -Cooper, Alan. _The Inmates Are Running the Asylum_. Sams Publishing, 2004. - -Spool, Jared M. _“Customers Request Solutions. We Need to Understand Their Problems.”_ Medium (blog), August 16, 2019. [Link](https://medium.com/@jmspool/customers-request-solutions-we-need-to-understand-their-problems-41db3b5c6d4d). diff --git a/topics/design/focus-product-development-on-customer-jobs-that-are-important-tangible-unsatisfied-and-lucrative.md b/topics/design/focus-product-development-on-customer-jobs-that-are-important-tangible-unsatisfied-and-lucrative.md deleted file mode 100644 index b88d1cfef..000000000 --- a/topics/design/focus-product-development-on-customer-jobs-that-are-important-tangible-unsatisfied-and-lucrative.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Focus product development on customer jobs that are important tangible unsatisfied and lucrative -description: Learn how to identify high-value customer jobs by focusing on important, tangible, unsatisfied, and lucrative needs to create effective value propositions that drive business success. -date: null -redirect: - - /2CRqzg ---- - -In value proposition design, focus your strongest efforts on customer jobs that fit in the middle of a Venn diagram comprising - -- Important: The customer's success or failure at completing the job leads to significant gains or pains -- Tangible: The results of success or failure are experienced immediately or often -- Unsatisfied: The customers' needs are not yet being met by existing value propositions in a satisfying way -- Lucrative: Many people need to complete the job, or a smaller number are willing to pay a premium to complete the job - ---- - -#### Related - -- [[¶ Jobs to be done]] - -#### Citations - -Osterwalder, Alexander. “The High Value Customer Jobs You Need to Focus On.” Strategyzer (blog). Accessed November 8, 2020. [Link](https://www.strategyzer.com/blog/the-high-value-customer-jobs-you-need-to-focus-on). diff --git a/topics/design/gestalt-principles-in-ui-design.md b/topics/design/gestalt-principles-in-ui-design.md deleted file mode 100644 index 343c3afb1..000000000 --- a/topics/design/gestalt-principles-in-ui-design.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Gestalt principles in UI design -description: Learn how Gestalt principles like Proximity, Similarity, and Closure improve UI design by organizing elements for better usability, clear visual hierarchy, and enhanced user experience. -date: 2020-07-13 -tags: - - design - - technique -redirect: - - /HPhrqg ---- - -![](assets/gestalt-principles-in-ui-design_261d02efd9a855cdb1eed8a98112d3ae_md5.webp) - -Take a second to look at the image above. How do you feel? Do you see that it is similar to an owl face? The answer to this phenomenon is called Isomorphism which is part of the 12 Gestalt rules. Just like Isomorphism, you will sometimes feel that a cloud is similar to a horse or a branding logo which is created by different shapes that can finally turn out to be a meaning object. Those logos embrace the brand mission, character, and business. So what are Gestalt Rules and how we can apply it to our UI Design? Let’s dive into it. - -## Introduction to Gestalt rules\*\* - -Gestalt is a word that is generated from Germany which means shape or form. Gestalt principles explain how humans perceive the outside world, how they recognize the pattern, and simplify complex images in daily life. In the User Interface Design field, we use Gestalt rules to make the design look more aesthetic and friendlier to users. In this article, we will discuss the 6 principles that are mostly used in every UI design. - -## Proximity - -Elements that are closer to each other are considered to be more related. The Principle of Proximity explains that people perceive elements in the way that they are grouped according to the distance. - -**Applying the Proximity Principle in UI Design** -If you are new to UI Design, there would be sometimes you are quite confused about how to arrange elements like heading, the subtitles, body text, or pictures… in a harmonious way so that it looks good and easy for users to find information. The Proximity principle will be a great guide for you to follow. For example, in an e-commerce product card, all the product detail content should be placed nearer to each other so that they become a group. So do the product description and the product review. Finally, we got 3 obvious different pieces of information: product detail, product description, and product review, which are ready to be scanned and digested. - -![](assets/gestalt-principles-in-ui-design_61289ca41b3b2f8dc1cae3cb3b9da6a7_md5.webp) - -The Proximity Principle is presented by white space to optimize and show the relationship, the visual hierarchy between components. Similar information and content are arranged near to each other so that it is perceived as a group. This way of organization creates good visual communication with users, helps them scan layout, read, and digest information quickly. Good applying of white space can enhance the interface appearance, support users to achieve goals faster as well as improve their user experiences. - -## Common fate - -Elements that go in the same direction as part of a stimulus is considered to be in a group. For example, when you see a flock of birds or a shoal of fish you will automatically feel that they are related to each other or belong to a group. - -**Applying Common fate Principle in UI Design** -The fact that the movement direction of elements shapes our perception of grouping creates the foundation for drop-down menu design. Let’s right click your mouse and see every sub-option of each category all appear on the right side of the main menu tab. This right orientation together with the highlight color effectively support users in their process of making choice. - -![](assets/gestalt-principles-in-ui-design_c25b23553a4fbc3e63707bb9a81d72c4_md5.webp) - -## Similarity - -Similar elements are usually considered to be in a group. If you are given 10 candies with 3 of them are red and the others are green. How would you group them? Most of us will have the answer that 3 red candies are in the same group and the other group will have 7 green candies. As a normal reaction humans tend to organize objects that have the same character or attribute to the same group. - -**Applying Similarity Principle in UI Design** -Similarity Principle is the answer to why the button system of every application is required to be unified. If you are familiar with an application, the time you take to finish a goal is only just a couple of seconds. The synchronization of the button appearance as well as the color throughout the application helps reinforce the smoothness of the user experience. - -Let’s imagine you are using an application, its button system changes in terms of color and style at every single stage to show the aesthetic skill of the UI designer. Think about it, if it is a money transfer application which requires high accuracy at each step. However, you have to stop to observe, think, and predict which button should you choose so that you won’t make any serious mistake that causes you to lose an enormous fortune. Feel the pressure and you will understand why there is a united button system in UI design. - -![](assets/gestalt-principles-in-ui-design_3a01b5fecbaf061c21599080993c6400_md5.webp) - -In website or application design the style, the color, as well as the character of the button are advised to be united so that users won’t get confused or take much effort to make a choice when interacting with your application or website. - -## Closure - -Our eyes will automatically fill in the missing parts between different shapes and turn them into a unified and meaningful whole. This principle is mainly used in designing icons and logos. The logo below is a well-explained example of the Closure Principle. The arrangement of those black shapes together with the white space successfully turned the unfinished panda image into a stunning and famous logo. - -**Applying the Closure Principle in UI Design** -The presence of the Closure Principle in UI design is shown through the iconography. Humans tend to recognize an image much faster than a line of text and the narrow space of mobile devices also contributes to the usage of icons. Therefore, icons are widely used in designing applications and websites especially applications where we need to communicate sharply and accurately in other to achieve our main goal. However, a label under the icon could be a great choice for those who are new to your product. - -![](assets/gestalt-principles-in-ui-design_13186fb397ff23cfe63db16a8ece4846_md5.webp) - -## Continuation - -The Continuation Principle explains that objects which are aligned with a straight or curved line usually are perceived to be related or in the same group. Take a look at the picture below, you can see that green dots which are on the right seem to be related than those on the left. - -**Applying the Continuation Principle in UI Design** -The continuation principle is best described in the UI field by the alignment of elements. If you have used an application to purchase things online you will see that all the content of a product card will be aligned to the left except for the call to action button. This organization draws our attention to the button which helps users quickly identify the important information, encourage the next step to happen, and reduces the frustration for users when there is a lot of information on one screen. - -![](assets/gestalt-principles-in-ui-design_0072fd89ad3c95ef57fb09a6299464ff_md5.webp) - -## Figure/ ground - -The Figure/Ground Principle demonstrates the human ability to visually separate the object from the background to place the focal point on the important element. This helps decrease our brain tension when it has to process too much information at the same time. - -**Applying the Figure/ ground Principle in UI Design** -There are a lot of ways to create the Figure/ ground effect, for example, semi-transparent overlay, shadow, or blurring the background. - -![](assets/gestalt-principles-in-ui-design_e25d1641f846de4167a75b8640b83744_md5.webp) - -## Conclusion - -User Interface isn't only about aesthetics. It's also about usability, performance, and how users experience the product along the way. - -Gestalt Principles will be an active supporter for us during our UI design process. Before we fully understand UI's beauty and how to create it, those principles are our guidance. - -However, never limit yourself to any line or principles. Rules are subjected to be broken. Feel free to stay creative. A great UI design is a harmonious combination of accessibility, feasibility, and art. diff --git a/topics/design/getting-started-with-webflow.md b/topics/design/getting-started-with-webflow.md deleted file mode 100644 index 73ad193f4..000000000 --- a/topics/design/getting-started-with-webflow.md +++ /dev/null @@ -1,171 +0,0 @@ ---- -title: Getting started with Webflow -description: Learn how to design responsive, high-quality websites quickly with Webflow’s no-code platform, featuring visual design tools, CMS, hosting, and easy animations for faster web development. -date: 2021-01-23 -tags: - - nocode -redirect: - - /LJ20vQ ---- - -![](assets/getting-started-with-webflow_5386df53f5360ba922dbe2c4b467dfdb_md5.webp) - -No-code platform has becoming a thing recently. With convenient and user-friendly workflow, no-code platform is a must to pick up for design and operation process, to shorten the development time and remove the misunderstanding between them with developers. - -## What is Webflow - -In short, Webflow is a web design tool, CMS, and hosting platform. Each aspect of the platform is represented by a particular product/feature set: - -### The designer - -A visual web design tool firmly grounded in web standards and best practices, the Designer translates design decisions into clean, production-ready code. Webflow was built to enable designers to develop websites familiarly — i.e., visually and effortlessly. - -If you’re mostly a prototyper, you can use the Designer alone. This function either helps sharing the prototype with devs to reproduce or exporting the code. - -But to fully utilizing Webflow, you’ll want to combine the Designer with the CMS and the Hosting features. - -### The CMS - -As the Designer, the CMS is a code-free web development tool. It has both in-Designer elements (where the site designer works) and on-site elements (where the client and/or content managers work). - -For now, know that in the Designer, the CMS lets you structure content types you’ll publish over and over again — like blog posts, product pages, etc. — by combining modular “fields.” Once you’ve created your content types, which we call Collections, you can use the Designer to determine how Collection items look on the site. - -### Hosting - -The final piece of the Webflow puzzle is the Hosting platform. Backed by Amazon Web Services (AWS) and Fastly, it’s blazing fast, super-reliable, and you’ll need it to enjoy some of the best features, including: - -- The CMS -- The Editor -- Form management -- Responsive images (using a device to automatically resizing images) -- Free SSL/HTTPS (for site security, which is a must for Google’s visitor permission) - -Okay, now that we have the lay of the land, let’s talk about diving in. - -## Setup before designing - -Let’s check the below image: - -![](assets/getting-started-with-webflow_aa6335d35ece7aa09c9484559131c433_md5.webp) - -Firstly, we’re recommended to fill in the default font, font size, and project name. The hosting, domain setup, embedded incode, SEO, Google analytic, and more benefits can be adjusted within the plans: - -![](assets/getting-started-with-webflow_15ed261870e05e4d45159ac062d63f5c_md5.webp) - -## Getting started - -### Toolbar - -![](assets/getting-started-with-webflow_b7f3ef4d478ea5cb2f99e218854ea861_md5.webp) - -Let’s start with the first image. From left to right: - -1. **Selector:** It is a component (which can create a component for all elements). "Container - grid" is a component which I named it. Considering about create a component in case you reuse that element regularly. -2. **Spacing:** Includes Padding and Margin. Padding is the space in the block; the margin is the outer space. Each container, text block, link block... can be adjusted for margin and padding depending on the purpose of use. -3. **Layout:** There are 5 types of Layout. Depending on usage needs, each layout helps us build our website differently. -4. **Typography:** We can input fonts from our laptop or directly from google font. You should check it out yourself; it’s easier than you think. -5. **Background, borders, and effects:** like in Figma or sketch, we can edit the background image, radius, have fun with shadows. - -### Navigator - -![](assets/getting-started-with-webflow_f8d5b754890ffc243e735a6e4fb21394_md5.webp) - -This point can be a red flag. In this case, we’re encouraged to create and name container components for regular-use ones. - -**_Tips:_** Name a component right after you create one. The component arrangement is the foundation of your website, especially responsive. It is the same as in Figma or sketch, a carefully arranged component is easier to edit, modify, and check back when needed - -## Design a website - -A website is divided into 3 parts: Header, Body, and Footer. - -### Header - -![](assets/getting-started-with-webflow_ab1951972be3533d03fe024933236f17_md5.webp) - -To add a Header, click “Cmd+K” and search for the keyword “NavBar” or look at the left corner. - -![](assets/getting-started-with-webflow_435ece2b3f4f240af0512011540e7010_md5.webp) - -NavBar is created with Brand, Nav Menu and Menu Button, located in one container. - -1. Brand: can be replaced with your company’s logo -2. Nav 3. Menu Button: I use a burger menu, but normally this could be a CTA. - -**_Tips:_** Check out [“How to made a Navbar on Webflow”](https://www.youtube.com/watch?v=vj-B5MBAjIc&t=495s&ab_channel=DesignPilot) on Youtube for more information. It might hard sometimes for beginners. - -![](assets/getting-started-with-webflow_ad25c3727e62ea25c77238725e166cf3_md5.webp) - -### Body - -![](assets/getting-started-with-webflow_523f7073ae646470be25042035c8e949_md5.webp) - -Example: Before doing design on Webflow, let’s define how these elements are grouped. The body should be divided into 4 areas. - -![](assets/getting-started-with-webflow_73ebf9dadeae2d52ba898a3fe0de296a_md5.webp) - -1. Grid and Layout - -![](assets/getting-started-with-webflow_683658131c557165d9023be3bbb4cf28_md5.webp) - -1. Default Grid with padding/margins - -![](assets/getting-started-with-webflow_2259389132637c4db847b8143770adf4_md5.webp) - -**_Tips_** - -- Divide your design’s layout, group them by div block -- Create a component for regular-use -- Create a large area (main container) to support div blocks in the website -- Responsiveness - -![](assets/getting-started-with-webflow_f8fbcb003ce6c8b919bc0cc7235b3110_md5.webp) - -### Footer - -![](assets/getting-started-with-webflow_868bfa2a53c561cff2296e7b08665242_md5.webp) - -![](assets/getting-started-with-webflow_fe50e0a67aa39fa34b8ba639a068d7cf_md5.webp) - -In this case, the grid layout is used to create an equal space for Div Block from 2 to 5, when Div Block No.1 needs a larger space. A Vertical direction can be used to help adjust this. - -![](assets/getting-started-with-webflow_483a7775cfee79b4bfb0d30f3cc2e5ea_md5.webp) - -This section is divided into 2 parts: - -1. Social link and one div block with height: 1px (_for line text_) -2. Email and phone number - -**_Tips_** - -- For clickable content, choose “Link Block” instead of “Text Block”. -- Layouts are critical to better create responsiveness. - -## Responsive - -![](assets/getting-started-with-webflow_d458119ad0184eee89c1d611a1543c50_md5.webp) - -It always starts with a Base breakpoint screen. When you do the responsive for a bigger screen or mobile, it’s much effortless. - -**_Tips_** - -- Images can be unexpectedly resized based on screen sizes. To prevent this, put images into a div block. The automatically responsiveness of div block can help maintain the images size. -- Layout, padding and margin can be adjusted based on screen sizes. In case the components position are changed, the whole process will be reset. -- The color, text style, font size, font-weight can be modified. - -## Animation - -You can play with default animation or create your own one. For more Page animation, this [youtube link](https://www.youtube.com/watch?v=69RRSEHWfCQ&ab_channel=Webflow) might help. - -![](assets/getting-started-with-webflow_05a4d54c84aaf4bbee7fca8473887937_md5.webp) - -Check out my **[full case study](https://kiwipay.webflow.io/)**. I added some effects at the burger menu and CTA: - -![](assets/getting-started-with-webflow_9ff9430476ddca3d674f41a53ec439ca_md5.webp) - -![](assets/getting-started-with-webflow_c1035a354ff1b911ba88d98c7815ad41_md5.webp) - -## Wrapping up - -Within a month of designing with Webflow, I can learn and practice much more to produce a more high-end website. Besides, I found out that this tool's Preview is sometimes unstable, which annoyed me, but the output was outstanding. - -Webflow is such a flexible tool, and now designers can work with design tools and a No-code platform for more fast and stunning achievement. I hope that these tips will spark some ideas and help you be a more efficient product designer. diff --git a/topics/design/good-design-understanding.md b/topics/design/good-design-understanding.md deleted file mode 100644 index 7b3bbeb2c..000000000 --- a/topics/design/good-design-understanding.md +++ /dev/null @@ -1,174 +0,0 @@ ---- -title: Good design understanding -description: Learn what makes good UX and UI design by exploring key factors like usability, accessibility, credibility, color, typography, and layout to create effective, user-friendly digital products. -date: 2018-04-02 -tags: - - design -redirect: - - /PAfUVA ---- - -Good design is not just what looks good. It also needs to perform, convert, astonish, and fulfill its purpose. It can be innovative or it might just get the job done. - -A good design cannot be measured by a finite way – multiple perspectives are needed. - -## What is a good UX design? - -User experience (UX) focuses on having a deep understanding of users, what they need, what they value, their abilities, and also their limitations. It also takes into account the business goals and objectives of the group managing the project. UX best practices promote improving the quality of the user’s interaction with and perceptions of your product and any related services. - -_Before taking a closer look on what is a good UX design, there are some examples for you to feel what is a good and bad UX design:_ - -- [Good design vs bad design example from everyday experiences](https://uxdesign.cc/good-design-vs-bad-design-examples-from-everyday-experiences-18a7d1ba002c)\* -- [5 example we can learn from bad design vs 5 example we can learn from good design](https://www.interaction-design.org/literature/article/bad-design-vs-good-design-5-examples-we-can-learn-frombad-design-vs-good-design-5-examples-we-can-learn-from-130706) - -## **The 7 Factors that Influence User Experience** - -![](assets/good-design-understanding_e75aeb9b90122499635bddc51805aca3_md5.webp) - -Good design is not just what looks good. It also needs to perform, convert, astonish, and fulfill its purpose. It can be innovative or it might just get the job done. - -A good design cannot be measured by a finite way – multiple perspectives are needed. - -### Useful - -A new product should first be designed to fulfil the users' needs. - -If the design doesn’t help users to successfully achieve what they need, it’s definitely not a good design. It doesn’t matter how good it looks if it doesn’t even accomplish the main purpose of its existence. - -### Usability - -A product should be designed that it is easy to use. If it takes so much time for the users to get what they want out of the product, they would be likely to find another product with less struggles to deal with. - -Improving the ease of use of the product is the work of interaction designers. It is including the ease of learning and efficiency. - -- Easy to learn refers to the user in contact with a new product, it is best to learn it easily and naturally. -- The content of efficiency includes easy manipulation (Fitts's Rule), simple steps (the number of clicks to complete the task), clear navigation (always know where they are, not lost). - -Designers in all three disciplines seek to create product features that are easily discovered and operated by the user. Usability engineers are concerned with aspects of the user experience, that include: - -- Learnability: Can users easily learn how to operate the product, and can they remember how to perform tasks when they return to the product the next time? -- Consistency: Are product features clearly and consistently labeled? -- Efficiency and effectiveness: Can users perform tasks with a minimal amount of effort and achieve their goals successfully? - -### Accessibility - -Accessibility is the ability to access (i.e., use and/or interact with) a product or service. In the design context, accessibility means that a product or service should be able to be used by everyone, regardless of a person’s physical, economic or cultural status. Studies have shown that accessible design benefits not only users with disabilities but everyone. - -There are four main types of impairment that will commonly affect digital projects: sight, hearing, touch and cognitive. Don’t rely on one sense alone to make your product or feature usable, but instead allow multiple forms of interaction and communication where possible — for example enabling text-to-speech functions for visually impaired users. - -### Desirability - -Are We Solving for the Right Pain Point? - -A test for desirability focuses on whether your solution is a nice to have or a must have for your customer. Ask yourself, - -- What task are you helping my customer complete? -- What does successful completion of that task look like for them? - -If you are solving the key pain points they encounter when trying to complete this task, your solution has met the test for desirability. If not, and there are other pain points that you haven’t addressed, then pivoting your solution might put you on a better path. - -To achieve ‘desirability’, design should be modeled on the following three levels of cognitive and emotional processing: - -- **Visceral** design is about how things look, feel and sound. -- **Behavioral** design is about how products function. The pleasure and effectiveness of use. -- **Reflective** design is all about the message, culture, and meaning of a product and its use. - -### Credibility - -Credibility relates to the ability of the user to trust in the product that you’ve provided. Not just that it does the job that it is supposed to do but that it will last for a reasonable amount of time and that the information provided with it is accurate and fit-for-purpose. - -It is nearly impossible to deliver a user experience if the user thinks the product creator is a lying - a clown with bad intentions – they’ll take their business elsewhere instead. - -Here are Jason Cranford Teague’s nine design principles for creating credibility-based user experience: - -- **Keep promises:** Inconsistent interfaces are a broken promise. Don’t make promises you can’t keep. -- **Show results:** Keep users in the know wherever possible through designs that respond intuitively to user input. -- **Know your voice:** Show your design skill in a consistent manner. A clear voice can accomplish anything. -- **Respect context:** Context in user experience prevents confusion. If possible, understand where, when, how, and on what device users will interact with your work and design accordingly. -- **Transition change:** If you don’t change transitions effectively users will lose their place and not trust you. -- **Guide, don’t dictate:** When you’re looking for a gorilla you’re often going to miss other events. Don’t fall for the sleight-of-hand. -- **Show, then tell:** We see patterns first, so offer visual identification followed by an explanation. Present a photo with a statement, for instance. People are more likely to believe it’s true. -- **Make it simple, not simplistic:** Reduce the amount of thinking a user needs to do whenever possible. -- **Always leave them wanting more:** ’nuff said. - -[Trustworthiness in Web Design: 4 Credibility Factors](https://www.nngroup.com/articles/trustworthy-design/) - -### Findability - -Findable refers to the idea that the product must be easy to find and in the instance of digital and information products; the content within them must be easy to find too. If you cannot find a product, you’re not going to buy it and that is true for all potential users of that product. - -![](assets/good-design-understanding_77e7ee12b43f15d452c133150959dd8f_md5.webp) - -### Value - -Value refers to being able to provide a user experience that is enriching the lives of your consumers. - -Finally, the product must deliver value. It must deliver value to the business which creates it and to the user who buys or uses it. Without value, it is likely that any initial success of a product will eventually be undermined. - -## What is a good UI design - -![](assets/good-design-understanding_e474e00ace9a524ebdff60d52f5e944c_md5.webp) - -User interface (UI) design is the design of user interfaces for software or machines, such as the look of a mobile app, with a focus on ease of use and pleasurability for the user. UI design usually refers to the design of graphical user interfaces—but can also refer to others, such as natural and voice user interfaces. - -### Color - -**60–30–10 Rule** - -> 60% is your dominant hue, 30% is secondary color and 10% is for accent color. - -This formula works because it creates a sense of balance and allows the eye to move comfortably from one focal point to the next. It’s also incredibly simple to use. - -**Color meaning** -Scientists have studied the physiological effects of certain colors for centuries. Besides aesthetics, colors are the creators of emotions and associations. The meaning of colors can vary depending on culture and circumstances. - -- **Red:** Passion, Love, Danger -- **Blue:** Calm, Responsible, Safe -- **Black:** Mystery, Elegance, Evil -- **White:** Purity, Silence, Cleanliness -- **Green:** New, Fresh, Nature - -**Contrast** -Color contrast is a key part of any visual composition. It brings the individuality for each UI element and makes all of them noticeable. User interfaces containing only shades from the same color family have fewer chances to draw users’ attention. Moreover, copy content in this UI will look illegible which make the interactions with a product almost impossible. - -### Typography - -**Readability** -Readable text affects how users process the information in the content. Poor readability scares readers away from the content. On the other hand, done correctly, readability allows users to efficiently read and take in the information in the text. You want users to be able to read your content and absorb it easily. - -- _Hierarchy_ defines how to read through content. It shows the user were to start reading and where to read through. It differentiates headers from body text. -- _Contrast_ is the core factor in whether or not text is easy to read. Good contrasts will make text easy on the eyes, easy to scan quickly, and overall more readable. On the other hand, poor contrast will force the user to squint and make reading the body text almost painful, not to mention a lot slower. -- Line height is a very common term meaning the space between individual lines of text. Line height is another factor in the readability of body text and even headers. -- Letter Spacing Like line height, letter spacing affects readability in Web typography. Letter spacing is, as the name suggests, the space between each letter in words. In print layout, negative letter spacing is a common technique to add a more fun feel to the layout, but it should never be used in body text. In any text, letter spacing is an obvious factor in legibility. -- Line length is, of course, the number of words per line. A good line length is one that allows the reader’s eyes to flow from the end of one line to the beginning of the next very easily and naturally. -- Font size. The minimum font size you should be using is 12pt, with a good reading size around 16pt (1em). Make text too small and users will be straining to see what it says, especially on lower density screens which aren’t that sharp. - -### Layout - -Layout is the visual organization and composition you give to all the visual objects that make up your design. - -The layout and design holds power with your audience at two levels: - -- **Look-&-Feel** - Turn-off a visitor with a negative "first impression" and you have lost them, with very little chance of reversing their impression of your brand. Present a professional design to your visitor and the inference is that you will do all you do in the same manner. -- **Usability** - Layout and design have definite “usability” ramifications. How the eye moves around a visual composition largely determines visitor behaviors in reaction to the web page. - -Layout-&-design has three primary goals: - -- Speaks to and connects with the target audience -- Supports site's content and message -- Drives desired visitor actions - -**Some layout rules:** - -- Grid -- White Space -- Emphasis and Scale -- Balance out elements - -### Consistency - -Consistency is a key principle in life and in design. Without it we can’t get far. Even the mightiest of problems will fall if you keep hacking it everyday! - -Benefits of consistency: - -- Users will learn faster how to use your design diff --git a/topics/design/grid-and-layout.md b/topics/design/grid-and-layout.md deleted file mode 100644 index f0a569c84..000000000 --- a/topics/design/grid-and-layout.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -title: Grid and layout -description: Learn how to create strong digital design layouts using grids, columns, gutters, and responsive techniques to make your graphic and web designs look clean and professional. -date: 2020-08-17 -tags: - - design - - technique -redirect: - - /zZgSqw ---- - -To almost fresher who started in Graphic design and Digital design, who wants to make our model look beautiful and right. When we - fresher receive brief directly from the client or leader, we start to search online for ideas. Honestly, those projects look pretty good, super clean, and all you want is your project look the same way as them, but you just CAN’T. That’s because you don’t do the same way they did. You might forget or skip the step called: “Build Layout.” - -## What is a layout and why it’s so important? - -Here are some definitions I read online: - -- **The layout** is the universal design tool. To layout, a page means to use photos, typography, space, icon, and ingredients to make a design own its story. -- **Layout** refers to the arrangement of elements on a page usually referring to the specific placement of image, text, and style. Understanding the **layout** of **designs** is very important. ... Proper **layout** enhances the look of the particular object and the objects as a whole piece of **design** in order to create a strong composition - -Let’s make it simple. Either you want your design looks like this: - -![](assets/grid-and-layout_3bdce6d13717bd0834f92c4a79970069_md5.webp) - -Or this - -![](assets/grid-and-layout_36c8e99a111f6eed2d177b9b0eb8228f_md5.webp) - -There are several things that make your work turn into “not good looking design” and one of that is you didn’t choose and prepare your layout good enough. (It happened to me once when I first started working as a visual designer). - -There are 2 types of layout: - -- Grid layout -- Free - formed layout - -## Grid - -A grid is a network of lines. The grid lines typically run horizontally and vertically in evenly spaced increments, but grids can be angled irregular or even circular. A secret to strong layouts helps designers organize the information with hierarchy and others. It’s a guide, not a strict ruleset. - -### Grid in web and app design - -As a fresher in digital design, this is what I’m understanding about Grid. There are 3 types of a grid: - -- Row -- Column (I use this mostly) -- Grid - -With **Row** and **Column,** we mostly have 12 counts, margin, width, gutter. - -### Field elements - -It's a block of design, whether that be text, image, or both. Background colors don't count as field elements unless they are a container for your text/image. I've seen the name field element be interchangeable with units, elements, parent containers — they're all the same. - -![](assets/grid-and-layout_1d7ff36ad43140f04c22a1344327120a_md5.webp) - -### Columns - -Columns are the thick colored blocks that make up the content width of your design. Field elements are to sit on a certain number of columns. Traditionally in a design system, the column width doesn’t change. But the number of columns varies from 12 on a desktop or 8 on a tablet and 4 on mobile. You can use anything you want, but most grids have 60–80px column widths. Choosing a column width that works for you is the most important since it’s the main determinant of your content width. - -![](assets/grid-and-layout_910a469e5bc586180f41f88680bba4bd_md5.webp) - -### Gutters - -Gutters are the space between the columns. 20px is a standard gutter size, and this spacing will be critical when you have a masonry design or a grid of card elements, a simple example being a photo gallery. - -![](assets/grid-and-layout_024c62923ce47df299bd7ee36aa26a77_md5.webp) - -### Side margins - -Side margins on mobile are usually 20–30px, and vary a lot between tablet and desktop. Whatever you choose as the side margin, will be the minimum white space you allow when you shrink your browser. When you expand your browser from this point, there will be white space until the next breakpoint. - -![](assets/grid-and-layout_a043920fee14ac284854524a4e074fd1_md5.webp) - -## How to use grid in the right way - -I’ve learned a lot from my leader how to apply the grid in design. We can’t just put elements inside the grid in a messy way. There are rules for it. - -### Field elements must sit on some number of columns - -![](assets/grid-and-layout_99ba657b5b121158b61df8f51c3935df_md5.webp) - -I mean, not all elements. As you see, just only the container must be between columns; the number of columns depended on the container’s size. It’s easier for the development team to code if we follow this rule, in case of force majeure (only if it make our UI better), we can create an exception. - -### Do not leave field elements in the gutters - -![](assets/grid-and-layout_c7cd9400e4b6a7c96f2df3c7f24791f9_md5.webp) - -![](assets/grid-and-layout_7417c8675f9f592d8b8fee0d635848e8_md5.webp) - -Your elements should sit within the columns and not be bleeding into the gutters. You CANNOT leave things in the gutters, that defeats the purpose of the grid. - -### It’s okay to nest elements inside fields that don’t align to the grid, as long as the parent field itself sits on columns - -![](assets/grid-and-layout_7df1c65ee21b0bfe2452413491f8f9bf_md5.webp) - -Sound weird right, everything I wrote about seems nonsense here. But It’s a trick to me, whenever you want to put an element that not fit to your grid (you should put them into the grid. But in case don’t do it make your design look better), all you need is creating a container fitted into a grid and put your elements into them. - -The Dev team always want our design put into a grid to save their time coding, with this little trick, we're no longer in a war-zone with them - -### Do not use a column as outside padding unless intentional - -![](assets/grid-and-layout_4052e7ff934548603afc4e9abb43548d_md5.webp) - -![](assets/grid-and-layout_a106abd338fb3f0957d15d343faef1c5_md5.webp) - -We actually don’t need to indented 1 column each side, it will make our main images smaller, that’s the margin jobs already. - -## How these work in responsive - -In a traditional design grid, the column widths and gutters stay the same, just the number of columns change. Why? And how does that work? This was to make things easier when you designed. If a set of three cards sat on 4 columns each on desktop, you would show two cards on a tablet and wrap the third one so that it’d show on a second row. Yay! You didn’t need to do any resizing, because you already knew that it sat on four columns. On mobile, the answer is easy too, you would show one card, and the rest stacked beneath it. If you wanted, you could also get creative and choose only to show one card on mobile or do a horizontal scroll. These breakpoints are the point of reference in code. - -![](assets/grid-and-layout_f0045709ef2c31affbb508461c5229cd_md5.webp) - -## Fixed & fluid grid - -[A simple video](https://youtu.be/T6MCkGWSXa0) to further understand Fixed & Fluid grid - -### Fixed grid - -If your developer codes a fixed grid when you shrink from desktop to tablet, you’ll get to the next breakpoint, and there will be lots of side margins shrinking until the next breakpoint. The text doesn’t wrap, and images don’t dynamically change. If your developer is not careful in making sure all sizes are accounted for, there could be a missed breakpoint, and your designs might look cut off (hopefully this doesn’t happen). But wait as soon as you hit that 768px breakpoint, the design will snap into place, and things will look right for the tablet. If you go smaller than that, the same thing will happen, your design will look the same until you reach another breakpoint - -### Fluid grid - -Now comes the beauty of fluid grids! As you shrink the window, things will change dynamically, your text is wrapping, and elements are getting narrower. However, these elements of yours still won’t change layout until you hit the next breakpoint that you designed. - -And that’s it. I hope after reading this, you will have some sense to make your design look great started with good layout and grid diff --git a/topics/design/jtbd-job-interview-structure.md b/topics/design/jtbd-job-interview-structure.md deleted file mode 100644 index 3c86a0ed4..000000000 --- a/topics/design/jtbd-job-interview-structure.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Jtbd job interview structure -description: Discover a step-by-step interview guide to uncover customer jobs-to-be-done, including how to explore tasks, needs, feelings, and circumstances for better product and service insights. -date: null -redirect: - - /YZVVUg ---- - -This interview structure can be used to uncover customer jobs-to-be-done. - -1. Get background about the participant and the job. (Build rapport) - 1. Tell me a little about yourself and what you do - 2. When was the last time you did the main job - 3. How did you feel overall while getting that job done -2. Understand the main job and related jobs (talk about JTBD) - 1. What are you trying to accomplish? What tasks are involved? - 2. What problems are you trying to prevent or resolve?  - 3. What helps you achieve your goals? - 4. What would the ideal service be to do the job for you? - 5. What else are you trying to get done? -3. Understand the process of executing the job (go through stages of getting the job done) - 1. How do you get started? - 2. What is the previous step? What is the next step?  - 3. How do you continue you after that?  - 4. How do you make decisions along the way? - 5. How do you feel at each stage in the process?  - 6. How do you know you are doing the job right?  - 7. How do you wrap things up?  -4. Find needs (uncover desired outcomes)  - 1. What workarounds exist in your process? - 2. What do you dread doing? What do you avoid? Why? - 3. What could be easier? Why? - 4. Why do you avoid doing certain parts of the job?  - 5. What’s the most annoying part? Why is that frustrating?  - 6. How do you feel when the job is completeD? -5. Probe on circumstances (Find out when and where performing the job makes a difference and uncover the salient factors that frame getting the job done) - 1. Ins which situations do you act differently? - 2. What conditions influence your decisions? - 3. How do the environment and setting affect your attitude and feelings while getting the job done?   - ---- - -#### Related - -- [[¶ Persona]] -- [[Methods to overcome the limitations of personas]] - -#### Citations - -Kalbach, Jim, and Michael Schrage. _The Jobs To Be Done Playbook: Align Your Markets, Organization, and Strategy Around Customer Needs_. New York: Two Waves Books, 2020. diff --git a/topics/design/methods-to-overcome-the-limitations-of-personas.md b/topics/design/methods-to-overcome-the-limitations-of-personas.md deleted file mode 100644 index b8a1ee2e1..000000000 --- a/topics/design/methods-to-overcome-the-limitations-of-personas.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: Methods to overcome the limitations of personas -description: Explore the limits of traditional personas and discover alternatives like characters, thinking styles, and Jobs to be Done that provide deeper user insights and richer context for product design. -date: null -redirect: - - /myobfQ ---- - -Personas have a number of shortcomings: - -- [[Personas alone can't help teams achieve empathy]] -- [[Personas lack rich context]] -- [[Personas may create space for assumptions]] - -In response to these limitations, a number of alternatives have been proposed: - -- Alan Klement suggests that "personas" be replaced with "characters." Characters represent a subtle adjustment—Klement suggests that you need not even shift your vocabulary—but shifts focus toward a kind of story world that the personas inhabit, highlighting the anxieties, motivations, and personal histories of each persona that may have led them to your product. -- Indi Young promotes her concept of "thinking styles" instead of personas. Thinking styles represent users' patterns of thought, emotional reactions, and guiding principles. Gathered through listening sessions—qualitative research conducted deep in the problem space—thinking styles focus more heavily on rich contextual information than traditional personas. - -In addition, a large body of work has emerged around the Jobs to be Done framework. There are at least two different interpretations of jobs to be done. However, both shift focus away from individual people interacting with the product and toward the need that the product needs to fulfill. Its focus is on the process an individual might undertake to resolve some tension their life. These "jobs" can be simple or complex, but they're relatively stable over time (like Cooper's goals) and should be solution agnostic. - -- [[¶ Jobs to be done]] - ---- - -#### Related - -- [[§ Personas]] -- [[¶ Persona]] - -#### Citations - -Cooper, Alan. _The Inmates Are Running the Asylum_. Sams Publishing, 2004. -Klement, Alan. _“Replacing Personas With Characters.”_ Medium (blog), September 2, 2017. [Link](https://medium.com/down-the-rabbit-hole/replacing-personas-with-characters-aa72d3cf6c69). -Spool, Jared M. _“Customers Request Solutions. We Need to Understand Their Problems.”_ Medium (blog), August 16, 2019. [Link](https://medium.com/@jmspool/customers-request-solutions-we-need-to-understand-their-problems-41db3b5c6d4d). -Young, Indi. _“Describing Personas.”_ Medium (blog), August 13, 2020. [Link](https://medium.com/inclusive-software/describing-personas-af992e3fc527). diff --git a/topics/design/omit-needless-demographic-information-when-creating-personas.md b/topics/design/omit-needless-demographic-information-when-creating-personas.md deleted file mode 100644 index 1be31ebdf..000000000 --- a/topics/design/omit-needless-demographic-information-when-creating-personas.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Omit needless demographic information when creating personas -description: Learn why demographic information in user personas often hinders empathy and how focusing on user needs can improve digital product design without unnecessary details. -date: null -redirect: - - /embV8Q ---- - -Demographic information is a common feature of user personas. But often this information is unnecessary: demographic information rarely has an effect on the user's thought process, needs, or desires from a digital product. Moreover, demographic information—even photos and names—can provoke "anti-empathy," assumptions, and other subconscious responses among team member. There are some situations in which demographic information is important to design work, but more often than not it adds unnecessary clutter to what should serve to inspire empathy. - ---- - -#### Related - -- [[§ Personas]] -- [[Personas alone can't help teams achieve empathy]] -- [[Personas lack rich context]] -- [[Personas may create space for assumptions]] diff --git a/topics/design/people-fill-information-gaps-with-assumptions.md b/topics/design/people-fill-information-gaps-with-assumptions.md deleted file mode 100644 index db7e387f5..000000000 --- a/topics/design/people-fill-information-gaps-with-assumptions.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: People fill information gaps with assumptions -description: "Learn how the mind fills gaps in information with assumptions and stories, leading to quick but often inaccurate judgments due to the 'what you see is all there is' bias." -date: null -redirect: - - /hMmNvw ---- - -Nature abhors a vacuum; the mind abhors a gap in information. When people receive information that is incomplete in some way, they will try to fill the gaps with their own assumptions. We too easily decide that we know things, even when gaps remain; we skim past the gaps in our knowledge and fill in the gaps with myths and stories that help overcome our uncertainty. We make up stories and pretend they are true. - -Daniel Kahnemann describes the phenomenon of "what you see is all there is," a cognitive bias that describes how we form opinions based on the information that is readily apparent. We gloss over information that we don't have and therefore come to conclusions too quickly. Often, we tell ourselves stories to help fill the gaps. We shape the information we do have into a coherent narrative that overcomes the missing pieces. - -This is an example of how System 1 thinking supersedes System 1 thought. It enables us to make judgments quickly, but these decisions may not be well-informed. - ---- - -#### Related - -- [[Narratives generate real-world effects]] -- [[Rationalistic models of human behaviour are limiting]] -- [[Decontextualize ideas to make them new]] - -#### Citations - -- [Varol - Think like a rocket scientist](https://publish.obsidian.md/mobydiction/Varol+-+Think+Like+a+Rocket+Scientist) diff --git a/topics/design/personas-alone-can-t-help-teams-achieve-empathy.md b/topics/design/personas-alone-can-t-help-teams-achieve-empathy.md deleted file mode 100644 index 4b2d134ea..000000000 --- a/topics/design/personas-alone-can-t-help-teams-achieve-empathy.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Personas alone can t help teams achieve empathy -description: Discover why personas alone may not build true user empathy in design teams and how relying on them can create a false sense of user understanding in product development. -date: null -redirect: - - /33cgpw ---- - -Personas are frequently cited as helping development teams achieve empathy for their users. - -It's worth noting that achieving empathy isn't as easy as some might suggest. It's means much more than just considering the user's perspective; it requires a concerted effort to imagine the world through the user's eyes. Real empathy requires effort and even training. - -Personas are intended to facilitate that; yet, personas are sometimes constructed with little thought or care as to why or how they should actually be crafted and used. They've just become part of the process—almost a ritual more than a tool. In fact, it's questionable whether or not personas facilitate empathy at all. According to one (admittedly limited) academic study, personas didn't help its student participants overcome "egocentric anchoring": the tendency to use the self as the primary reference point for evaluating the perspectives of others. The students reduced the personas down to a handful of core attributes, and fixated most heavily on those aspects that marked the personas as "different"—hardly putting themselves in the shoes of their users. When considering personas that they perceived as being part of their "in group," the students defaulted back to their own needs as their primary reference.4 Particularly elusive was "affective" empathy—an emotional connection to the user being described by the persona. - -There is also the risk that the creation and use of personas provides the design team with false assurance that "empathy" has taken place, even if it hasn't. In other words, the persona becomes a box to check so the team can performatively convince itself that it's doing user-centric work. - ---- - -#### Citation - -Haag, Maren, and Nicola Marsden. _“Exploring Personas as a Method to Foster Empathy in Student IT Design Teams.”_ International Journal of Technology and Design Education, April 24, 2018. [Link](https://doi.org/10.1007/s10798-018-9452-5). diff --git a/topics/design/personas-communicate-research-or-they-communicate-assumptions.md b/topics/design/personas-communicate-research-or-they-communicate-assumptions.md deleted file mode 100644 index 1c3670bf5..000000000 --- a/topics/design/personas-communicate-research-or-they-communicate-assumptions.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Personas communicate research or they communicate assumptions -description: Personas should be based on user research to provide accurate insights, while proto-personas rely on assumptions that require careful validation to avoid misleading design decisions. -date: null -redirect: - - /dRi2EA ---- - -[[§ Personas]] should communicate insights generated through user research. If they contain information not uncovered through research, that information should be treated as an assumption and subject to research and evaluation. - -Teams often create proto-personas as a proxy for persona research; these should be approached with caution, as the fact that they are based on assumptions or guesswork may get lost in transmission, or overlooked when design work begins in earnest. - -Personas in general are a form of narrative. This means that they must be approached carefully, as [[narratives generate real-world effects]]. - ---- - -#### Related - -- [[Focus personas on tasks, goals, and motivations]] -- [[Personas document shared understanding about users]] -- [[Personas may create space for assumptions]] -- [[Personas lack rich context]] -- [[People fill information gaps with assumptions]] -- [[¶ Persona]] -- [[§ Personas]] - -#### Citations - -Harley, Aurora. _“Personas Make Users Memorable for Product Team Members.”_ Nielsen Norman Group (blog). Accessed October 10, 2020. [Link](https://www.nngroup.com/articles/persona/). - -Laubheimer, Page. _“3 Persona Types: Lightweight, Qualitative, and Statistical.”_ Nielsen Norman Group (blog). Accessed October 12, 2020. [Link](https://www.nngroup.com/articles/persona-types/). diff --git a/topics/design/personas-document-shared-understanding-about-users.md b/topics/design/personas-document-shared-understanding-about-users.md deleted file mode 100644 index 51b64e277..000000000 --- a/topics/design/personas-document-shared-understanding-about-users.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Personas document shared understanding about users -description: Personas help design teams understand user goals and priorities, prevent feature creep, and create shared empathy by focusing on real users’ needs throughout the product development process. -date: null -redirect: - - /LC4GBw ---- - -Whether or not personas can help design teams achieve empathy for their users, personas may still be worthwhile as a document that shares a team's current understanding of their users' perspectives, values, and priorities. While affective empathy can be difficult to achieve, cognitive empathy—consider another person's more functional needs—seems more attainable. Personas can be an effective shorthand for that purpose, if nothing else reminding product or design teams that there is a real, human user on the other side of the experience whose goals may differ from their own. - -This can keep teams manage the scope of their project. Alan Cooper suggests that personas can help prevent feature creep, as well as helping teams focus on user goals rather than features. Personas therefore may have utility as focalizing tools that, if used effectively, help limit design changes and establish a shared understanding and vocabulary about users. - ---- - -#### Related - -- [[Personas alone can't help teams achieve empathy]] -- [[¶ Persona]] - -#### Citations - -Cooper, Alan. _The Inmates Are Running the Asylum_. Sams Publishing, 2004. - -Haag, Maren, and Nicola Marsden. _“Exploring Personas as a Method to Foster Empathy in Student IT Design Teams.”_ International Journal of Technology and Design Education, April 24, 2018. [Link](https://doi.org/10.1007/s10798-018-9452-5). - -Harley, Aurora. _“Personas Make Users Memorable for Product Team Members.”_ Nielsen Norman Group (blog). Accessed October 10, 2020. [Link](https://www.nngroup.com/articles/persona/). diff --git a/topics/design/personas-lack-rich-context.md b/topics/design/personas-lack-rich-context.md deleted file mode 100644 index 2dea93ae4..000000000 --- a/topics/design/personas-lack-rich-context.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Personas lack rich context -description: Learn why personas often lack deep context and how missing background can lead teams to make incorrect assumptions, causing misalignment in design and product development. -date: null -redirect: - - /0qbATA ---- - -[[§ Personas]] are popular design artifacts. But, they can be hollow. While they often provide some basic attributes, tasks, or descriptors, they often lack rich contextual information that explains why the persona's need arose in the first place. - -This can be problematic. As Alan Klement argues, in the absence of this information, team members may fill in the information themselves—whether they are aware of it or not. This creates conditions for misalignment within the team. - ---- - -#### Related - -- [[Narratives generate real-world effects]] -- [[People fill information gaps with assumptions]] - -#### Citations - -Klement, Alan. “Replacing Personas With Characters.” Medium (blog), September 2, 2017. [Link](https://medium.com/down-the-rabbit-hole/replacing-personas-with-characters-aa72d3cf6c69). - -Spool, Jared M. “Customers Request Solutions. We Need to Understand Their Problems.” Medium (blog), August 16, 2019. [Link](https://medium.com/@jmspool/customers-request-solutions-we-need-to-understand-their-problems-41db3b5c6d4d). diff --git a/topics/design/personas-make-abstract-research-concrete-for-design-teams.md b/topics/design/personas-make-abstract-research-concrete-for-design-teams.md deleted file mode 100644 index ff65ee42f..000000000 --- a/topics/design/personas-make-abstract-research-concrete-for-design-teams.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Personas make abstract research concrete for design teams -description: Personas help design teams focus on real user needs by creating detailed, memorable models of single users, improving product decisions without relying on personal assumptions. -date: null -redirect: - - /F4wFAg ---- - -The job of personas is to make abstract findings from research feel more concrete for teams working on a product or design. Distinct from market segments or user groups, personas provide a model of a single user that highlights specific details that the design team should keep in mind. They should encourage team members to consider end users and their real needs when making decisions about the design, rather than relying on their [personal expectations](https://publish.obsidian.md/mobydiction/notes/%C2%B6+Egocentric+anchoring). Personas should therefore include enough detail to be memorable, without clutter that won't impact the ultimate design. - -A name and photo may be included if they help the team remember the persona. However, [take care that this information is actually useful](https://publish.obsidian.md/mobydiction/notes/Omit+needless+demographic+information+when+creating+personas). - ---- - -#### Related - -- [[Omit needless demographic information when creating personas]] - -#### Citations - -Harley, Aurora. _“Personas Make Users Memorable for Product Team Members.”_ Nielsen Norman Group (blog). Accessed October 10, 2020. [Link](https://www.nngroup.com/articles/persona/) diff --git a/topics/design/personas-may-create-space-for-assumptions.md b/topics/design/personas-may-create-space-for-assumptions.md deleted file mode 100644 index a6621d6df..000000000 --- a/topics/design/personas-may-create-space-for-assumptions.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: Personas may create space for assumptions -description: Learn why personas can mislead design teams by including unnecessary demographic details and how focusing on real user research improves understanding and creates better user experiences. -date: null -redirect: - - /Xv9vQg ---- - -Personas have come under fire for including extraneous information that, beyond simply being unnecessary, can distract or contribute to misunderstandings about users and their contexts. For instance, Alan Klement suggests that by describing a set of attributes, personas distract teams from the actual problems their users are hoping to solve. - -Much of the critique of personas has centered around their inclusion of demographic information and photography to represent the user. Indi Young and Klement have, in their own ways, suggested that including demographic information can instigate subconscious reactions and biases in team members. They may therefore inspire "anti-empathy." At best, demographic data may provoke divergent understanding as different members of the team interpret even soft descriptors like "low-income," "middle-aged," or "college student" in different ways. This is not to say that there are never situations when this kind of information is useful; but, these are likely exceptions rather than the norm. - -Young, Klement, and others have therefore suggested adjustments to personas to mitigate the risk of biases. Much of this advice suggests removing demographic information or any data that would align the persona to a broad population rather than an archetype of an individual. For instance, Young suggests using gender-neutral names. However, because even those names may be suggestive of culture, race, or class, it may be preferable to refer to the "personas" with general descriptors rather than names at all. Young has also proposed using demographic data to disrupt assumptions people might have of a given group. - -Personas have also been criticized for being too reductive. This is, to some extent, unavoidable; personas are by intent and by design meant to condense research into an accessible format. Spool says that personas can be a valuable tool, but suggests that most teams create overly simplistic user models that don't actually add to the process. "They look good," he writes, "but they get ignored." - -It would be impossible to fully capture the complexity of every user and their context in a diagram that can easily be digested by a number of individuals. However, this creates the risk that the design team will fill in gaps with their own assumptions about the user. Klement writes that "[b]ecause personas focus on creating a story made up of a customer's attributes, instead of a story that explains a purchase, personas leave the brain in an unsatisfied state." This leads each member of the team to come up with their own, potentially different, story about the persona's needs and motivations. Teams should take care, then, to consider carefully whose experiences are represented by the personas and whose are left out. - -Personas may also become too strong a stand-in for actual user research. That is, once personas are established, they may be used as an excuse for not engaging in further contact with users. This is dangerous. Even the most expertly designed persona can provide only a rough approximation of the real lived complexity of another human being. We can't test designs with persona, nor can we gather meaningful insight from them. We need real people for that. A persona shouldn't be the end of research. It's represents just part of it the research work to be done. - -Better than any persona is ongoing exposure to actual users. - ---- - -#### Related - -- [[People fill information gaps with assumptions]] -- [[Personas lack rich context]] - -#### Related - -Klement, Alan. “Replacing Personas With Characters.” Medium (blog), September 2, 2017. [Link](https://medium.com/down-the-rabbit-hole/replacing-personas-with-characters-aa72d3cf6c69). diff --git a/topics/design/personas-start-with-qualitative-research.md b/topics/design/personas-start-with-qualitative-research.md deleted file mode 100644 index cc01bf3c0..000000000 --- a/topics/design/personas-start-with-qualitative-research.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Personas start with qualitative research -description: Discover how effective user personas emerge from qualitative research and sensemaking to capture rich user experiences, helping teams design better products by focusing on real user needs and behaviors. -date: null -redirect: - - /mcFQ6g ---- - -Personas shouldn't be created so much as discovered. First and foremost, personas document research. Personas articulate patterns of thought and feeling. These are attributes that can't easily be quantified. Effective user personas should be an outcome of qualitative research. Personas should be the result of [sensemaking](https://publish.obsidian.md/mobydiction/Sensemaking+MOC), not measurement. - -This is [[¶ Abductive reasoning | abductive research]]: it doesn't start with a supposition or hypothesis, but begins instead with an openness to evidence that can be assembled into a theory. Persons require attention to what [Christian Madsbjerg](https://publish.obsidian.md/mobydiction/notes/%E2%89%88+Madsbjerg+-+Sensemaking) calls "thick data"—the rich context below the surface that details what an experience feels like rather than simply noting that it occurs. Personas should describe not just what kinds of people use the product, but help teams understand how they understand their relationship to the product and the problem space it occupies. - -Due to a small sample size, it is possible that these qualitative personas will not completely capture a representative portrait of the use base. Some organizations therefore build on qualitative personas with another layer of quantitative research to help validate qualitative findings and identify statistical patterns among the users. This adds a veneer of scientism to personas and provides assurance that they are representative of the user base. However, the creation of statistical personas is far more time consuming and may yield only incremental benefits for the extra work.5 - -Teams and organizations will sometimes create personas based on their best current understanding of the user; these should be clearly described as _proto-personas_ and be _treated carefully as assumptions_ that should be subject to testing. Teams should be reluctant to base any significant decisions on proto-personas unless they're supplemented by more rigorous research. Nevertheless, when research isn't possible, proto-personas may be better than nothing. In the very least, they may serve as reminders that, at the end of the day, work needs to serve the customer. They may also be more useful in Lean UX environments where work will be continually tested and iterated upon even without initial discovery research. - -[Alan Cooper](https://publish.obsidian.md/mobydiction/Alan+Cooper) emphasizes the importance of selecting a primary persona—the user archetype whose needs must be met at all costs. Most projects will involve multiple personas, but one should be identified as most important. Cooper also suggests that there may be value in creating anti-personas—in other words, defining user for whom you explicitly won't design. - -Personas should be living, breathing documents. As knowledge about the user or customer evolves, so should the personas. - ---- - -#### Related - -- [[§ Personas]] -- [[Personas make abstract research concrete for design teams]] -- [[Personas document shared understanding about users]] -- [[Personas communicate research or they communicate assumptions]] - -#### Related - -Cooper, Alan. _The Inmates Are Running the Asylum_. Sams Publishing, 2004. - -Laubheimer, Page. _“3 Persona Types: Lightweight, Qualitative, and Statistical.”_ Nielsen Norman Group (blog). Accessed October 12, 2020. [Link](https://www.nngroup.com/articles/persona-types/). diff --git a/topics/design/the-principle-of-spacing-in-ui-design-part-1.md b/topics/design/the-principle-of-spacing-in-ui-design-part-1.md deleted file mode 100644 index 8a1b9cb7a..000000000 --- a/topics/design/the-principle-of-spacing-in-ui-design-part-1.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: The principle of spacing in UI design part 1 -description: Learn why spacing is key in design, affecting content, graphics, navigation, and overall aesthetics to create a balanced, clear, and engaging user interface. -date: 2018-01-18 -tags: - - design - - technique -redirect: - - /yjs57Q ---- - -## Why is spacing important? - -Before I talk about spacing principle, you should know how spacing important is with our design. - -In design have four main factors in such as: - -- Content -- Graphic -- Navigation -- Identify - -Spacing has not only an impact directly on this factors but also on aesthetics design. Even more, it gives the page a united harmonic look. It decides you want them to pay attention to what. If you know the principle of spacing, you can use it in all your products. And you have a power strength of designers. - -Now let’s check the below example: - -![](assets/the-principle-of-spacing-in-ui-design-part-1_cdf0d4d3eccb21431ec5e9c04b336c24_md5.webp) diff --git a/topics/design/the-principle-of-spacing-in-ui-design-part-2.md b/topics/design/the-principle-of-spacing-in-ui-design-part-2.md deleted file mode 100644 index 842f2a432..000000000 --- a/topics/design/the-principle-of-spacing-in-ui-design-part-2.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: The principle of spacing in UI design part 2 -description: Learn the best UI design spacing tips for vertical and horizontal elements, including line-height, paragraph spacing, input fields, and icon alignment to improve readability and user experience. -date: 2018-11-01 -tags: - - design - - technique -redirect: - - /YoRagA ---- - -## Vertical spacing - -### Spacing within each paragraph - -Firstly, I start the simplest content type — paragraph. One of the atoms decides aesthetic design. So you need to focus on them. You commonly depend on default line-height of the font to design the content. I have increased line by 2px to 3px in this way because they are too tight. - -Everything almost has a proportional so line-height also it. **1.5** is a suggestion for you to have a good starting point. - -![](assets/the-principle-of-spacing-in-ui-design-part-2_83a1ac346ab98c836ab637c14c3ac9cf_md5.webp) - -However, you should not use the same line-height for all text. Here is a common mistake for beginners. 1.5 is a great proportional for body text, but as heading or title, it should get tighter. I suggest that the text of the content will use the title of **1.2**, the subtitle of **1.3** and the body copy of **1.5**. - -![](assets/the-principle-of-spacing-in-ui-design-part-2_7edc02bcf0ab2561f68d9869b7addbac_md5.webp) - -### Spacing between two consecutive paragraphs - -I found a rule is to use paragraph spacing equal to font-size of the content using. By that, we can remember spacing easier. Besides, note you should use paragraph spacing instead of hitting enter to go down a line. As of spacing when you hit enter this is too larger, the paragraph will control white space easier between two consecutive paragraphs. - -### Spacing within the list items in a list - -When you design a list item with a multi data structure, you should not divide into the margins of the list the same. It won’t feel obvious or connected. The user has to work hard to interpret data and can misunderstand the meaning as of putting the wrong item. This solution is to split the space each a group into two formats size. - -### Spacing within input fields with labels - -You can see difference spacing between the two consecutive input fields in labels. - -In the left, if you equalize spacing for all input fields, it seems tight and not look actively bad. The label doesn’t have a visual hierarchy. Moreover, the user can’t quickly scan them. The problem with the first card is the spacing of that labels don’t have breathing room necessary and divide the system-level sizing. - -In the right, it seems perfect more and improves the legibility of the user. My approach is to start designing something with too much spacing, then remove it until you have an eye-catching design. You need to notice that you should have some difference between the spacing and divide the least three formats in your design (small, medium and large spacing) I mentioned it in [part 1)](https://medium.com/dwarves-design/the-principle-of-spacing-in-ui-design-part-1-3354d0d65e51). In addition, they can support you to define the system. **16px** is a great number to start because it divides nicely ( 4px = **16** x 0.25, 8px = **16** x 0.5, 12px = **16** x 0.75, …) - -You usually start to add a bit of white space. If something is too cramped, you will add a bit more spacing until everything looks better in your design. By this way, you only have a minimum spacing. So you need more space. This way will take a lot of your time and not have a good result. One of the ways to have an elegant design is your design should start with too much white space. - -## Horizontal spacing - -### Spacing inside components - -I suggested the bottom values of input and button to design better. You can see clearly with font-size 16px on phone screen or computer monitor. 16px of horizontal padding for both is a number that easy to remember when it defined in term of font-size. - -### Spacing two components - -When you design a form with labels, if spacing in labels is the same, the elements in the form group won’t explicitly associate. The user feels ambiguous about content. If there isn’t a visible separator, there isn’t obvious. - -### Spacing between icons - -It’s not only vertical spacing but also horizontal spacing it’s easy to make this mistake with components that are laid out horizontally, too: - -Whatever you want to connect a group of elements, you always make sure around the padding of the group is more than within it. - -### Spacing within components with icons - -I used 8px for spacing between components with icons. It is a great number that you can apply for any adjacent components and connect them together. You can choose another reasonable number — giving yourself the freedom to find a lot of easier to build a better UI design. - -## In conclusion - -- You will improve the readability/legibility of the user by respect information hierarchy, allow track and comprehend information more easily. -- You will have a strong spacing system with limited values and limited application rules -- Developers will become faster as they know all the rules of your spacing system. - -You can follow me on [https://dribbble.com/Anna23593](https://dribbble.com/Anna23593) and thank you for taking the time to read it. diff --git a/topics/design/three-clicks-rule-is-a-myth.md b/topics/design/three-clicks-rule-is-a-myth.md deleted file mode 100644 index 53bc828b2..000000000 --- a/topics/design/three-clicks-rule-is-a-myth.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Three clicks rule is a myth -description: 'The "three clicks rule" for website navigation is a myth, as research shows click count doesn''''t impact user experience or conversion rates, and more clicks can sometimes improve control and satisfaction.' -date: null -redirect: - - /TeJtTw ---- - -The so-called "three clicks rule," which stipulated that all information on a website should be accessible within three clicks, is a myth. This "best practice" originated in Jerry Zeldman's 2001 book _Taking Your Talent to the Web_. Researchers going as far back as 2003 have found that there's little correlation between the three clicks rule and conversion rate. Moreover, users don't necessarily perceive fewer clicks as being any easier to use. - -In fact, there are situations in which more clicks might deliver the better experience. More clicks provides a sense of control, and so they may be appreciated in risky or weighty transactions. As well, the limitations of the device must be considered, as well. - ---- - -#### Citations - -Goesch, Robert. “Stop Counting Clicks.” Boxes and Arrows (blog), October 12, 2020. [Link](https://boxesandarrows.com/stop-counting-clicks/). diff --git a/topics/design/three-levels-of-design.md b/topics/design/three-levels-of-design.md deleted file mode 100644 index 06c668e98..000000000 --- a/topics/design/three-levels-of-design.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: Three levels of design -description: Learn how Don Norman’s three levels of design—visceral, behavioral, and reflective—shape user experience and determine whether a product succeeds or fails. -date: 2018-03-21 -tags: - - design -redirect: - - /6IFpAQ ---- - -Don Norman in his book titled _The Design of Everyday Things_ talks about three levels of design — Visceral level, Behavioral level and Reflective level — on how they work together and play an important role in determining how people like or dislike the product. In other words, these three levels together determine whether your product is successful or not. - -## Visceral level - -Visceral level is about... - -### First impression - -[First impression is the best impression](http://en.wikipedia.org/wiki/First_impression_%28psychology%29). Working on building a great on-boarding experience helps you tell your users what your product is all about and make them like your product. It is about telling the story of why users should use your product. - -### Attraction\* - -The more attractive (pretty) your product is the more chances you are going to get users’ immediate attention. Its when you hear users say, “WoW! Look at that!” - -### Immediate emotional impact - -Immediate feeling users get looking at your product (color, design etc.,) or touching your product (it feels good…) or feeling the immediate need of the product! - -## Behavioral level - -Behavioral level is about... - -### User experience - -Bad user experience, no matter how good your product is, can cripple users’ expectations, drive anger & frustration which ultimately results in users not using your product. - -### Understanding how users use your product - -Its not just about building the user experience, it is also about understanding how users use your product, getting the right feedback so you can improve your product’s user experience. - -### Expectations - -It is about what users expect from your product. In other words, the product should deliver what it promises to the users. e.g., Dropbox should allow its users to put stuff in their Dropbox and get to it from their computers, phones, or tablets. - -## Reflective level - -Reflective level is about - -### Memories - -The joy users get using your product that lasts forever. - -### The relationship with the product - -- If users like your product, they are going to be more attached to it. They build a relationship with the product. They are proud to use it. -- If users do not like your product, they are not going to use your product. Period. - -### Overall impression of the product - -The more users like the product, the more they are going to keep using it. Not only use it, but recommend it to others. diff --git a/topics/design/ui-design-fundamental.md b/topics/design/ui-design-fundamental.md deleted file mode 100644 index 657607d98..000000000 --- a/topics/design/ui-design-fundamental.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: Ui design fundamental -description: Learn how to design effective user interfaces by focusing on aesthetics, content, and interaction to create user-centered apps and websites that engage and meet user needs. -date: 2018-03-20 -tags: - - design -redirect: - - /p4Hp0g ---- - -When you design UI for an app or a website, the first thing that the designer needs to determine the purpose of the product and the target audience. You should plan and calculate for your UI design. - -Creativity does not exceed the limits of design goals. - -Differentiate between customer and user: Put more on the role of the user than the customer - -Identify the user who will use your app, identify the behavior, user characteristics. - -So we need research: - -- What they need -- Their behaviors -- Their Preferences - -## What users perceive from UI - -USER INTERFACE: (has 3 main elements) - -## Asthetics - -You should reply this questions: Where/Why do you think that they are beautiful? You will see that each person has a different opinion. Aesthetics is only relative and bases on standards: layout, typography, color. So, a beautiful design is a design that …? - -Aesthetics changes over time, mostly comes from trends, If you can’t create trends, just follow them. Some current trends: - -Futuristic ornamentalism; Simplicity & Comfort; Extra Depth (with Semi-Flat Design); Custom Illustrations; Animations, Gifts & Cinemagraphs; Micro Interactions; Integrated Animations; Obnoxious Bright Gradients; Semi Realistic 3D; One Color 3D Design; 80s-90s Color Palettes & Patterns; Big Bold Typography; Creative Typography; Particle Background; Modular Blank; Split Page Design; Vibrant & Saturated Color Page; Mondrianism. - -## Content - -When you design a website or an app, you have to reply the question: What is this site about? Because the user not only look at your website but also read content. One of the parts the user look most is landing page, the content should follow the items below: - -- Relevant & Useful -- Accurate & Structure -- Credible & Finable -- Scannable & Simple. - -## Interaction - -Interaction design is specifically a discipline which examines the interaction (via an interface) between a system and its user. It may also incorporate design focused on how information should be presented within such a system to enable the user to best understand that information though this is often considered to be the separate discipline of “information design” too. - -![](assets/ui-design-fundamental_4718c084b78a2e641043dff2ffb94dad_md5.webp) - -### Visibility & simplicity - -**Visual hierarchy** is the order in which a user processes information on a page; its function in user interface (UI) design is to allow users to understand information easily. By assigning different visual characteristics to sections of information (e.g., larger fonts for headings), a designer can influence what users will perceive as being further up in the hierarchy. - -The visual characteristics that a designer can use to influence users’ perception of the information are: - -- **Size:** the larger the element, the more attention it will attract -- **Color:** bright colors are more likely to draw attention over muted ones -- **Contrast:** dramatically contrasted colors will catch the eye easily -- **Alignment:** an element that breaks away from the alignment of others will attract more attention -- **Repetition:** repeating styles can give the impression that content is related -- **Proximity:** closely placed elements will also appear related -- **Whitespace:** more space around elements will attract the eye toward them -- **Texture and style:** richer textures will attract more attention than flat ones - -When information design does not have a strong visual hierarchy, a user’s eye follows a predictable reading path. This path is culturally influenced, as it is connected to the standard reading direction of written text. In the Western world, two main left-to-right paths exist, which can be described as a Z and an F pattern. - -A designer has the opportunity to use visual hierarchy to reinforce these natural paths, or deliberately use visual characteristics to break such patterns so as to draw the viewer’s attention to a focal point. Thus, the successful manipulation of this hierarchy empowers designers to lead users, quite literally, along a cleverly devised visual journey to a goal. - -**Read more:** [https://www.interaction-design.org/literature/topics/visual-hierarchy](https://www.interaction-design.org/literature/topics/visual-hierarchy) - -**Readability** is whether an extended amount of text—such as an article, book, web page - is easy to read - -**Legibility** is whether a small burst of text such as a sign or a headline is instantly recognizable. - -**Differentiate dissimilar things** - -**Communicate in user language** - -### Predictability - -**Affordance** -Affordance describes all actions that are made physically possible by the properties of an object or an environment. A bottle screw cap affords twisting. A hinged door affords pushing or pulling. A staircase affords ascending or descending. - -[Don Norman](https://en.wikipedia.org/wiki/Don_Norman) introduced the term [perceived affordance](http://www.jnd.org/dn.mss/affordances_and.html)\*\*\*\*in his book named [Psychology of Everyday Things](https://www.amazon.com/Psychology-Everyday-Things-Donald-Norman/dp/B000HVS5DG) to refer to the actions a user perceives to be possible, distinct from those which are actually possible. - -Here is an [underlined text](https://uxdesign.cc/affordance-in-user-interface-design-3b4b0b361143#). At first, you might perceive it as a hyperlink but when you actually try to click it; it does not act like one. In this very case, it does not work like a hyperlink. It’s an example of how perceived and actual affordances could be distinct. - -Both actual and perceived affordances must be considered in design. - -Desired actions cannot be carried out if the object does not afford it, and afforded actions might not be carried out if the user does not perceive they are possible. - -A user’s perception and understanding of affordances might vary according to their ability, goals, beliefs, context and past experiences. A bottle screw cap may be a mystery to a person who has not encountered one before. A staircase may afford an able-bodied person to ascend to a higher floor, but a person with poor mobility can not afford the same action. - -Affordances can be deliberately constrained to enable only the correct or desired actions. A bottle screw cap cannot be tightened further when the bottle is sealed. A door with a plate instead of a handle cannot be pulled. A car steering wheel won’t turn any further when the wheels themselves are at the limit of their movement. - -In the physical world, the ridges or dots of a bottle cap provides a high-grip surface, suggesting some friction should be applied. A flat plate on a door suggests the door can be pushed in that place. The steering wheel connected to the top of steering column suggests it will turn around that point. - -**Mapping** -The following example will show how the individual bricks of the content type “event” can be mapped to different shapes of a “teaser” UI component in the target system (here: a website with desktop-like presentation). The teaser in the example is simplified and consists of four different text bricks, an image and metadata. (It’s the same for any other target UI component as with the teaser example.) - -![](assets/ui-design-fundamental_5c2e41286c6c2f47ccdbe589a50c4033_md5.webp) - -_Same content and different presentation: The three teaser variants extracted and isolated from the website with the individual UI bricks (except the image all exemplary available information is outputted textually)._ - -_The generic teaser variants schematized without content with the individual content bricks. The small teaser shows most information in this case and uses all exemplary content bricks of the content type “events”._ - -**Summarized:** One content type can be displayed in various shapes when the generic structure basically fits to the the generic UI structure. The UI components have just to be able to display or output the relevant bricks of the content type. The other way round it’s the same. Different content types can be displayed similarly (see following example). - -_Different content and same presentation: The basically different content types “event” and “article” are structurally different, but generically identical. Thus the bricks of both content types can be mapped to a generic teaser component that can display these bricks. In short: Different content types can be displayed identically. Which also means that different UI types can be served by content bricks from the same content type (see above)._ - -![](assets/ui-design-fundamental_2f86f2fad6448f61bf19fa256a25c973_md5.webp) - -## Conclusion - -Intentional Design: Interface without User is not User Interface - -3 pillars of UI Design: - -- Aesthetics -- Content -- Interaction. diff --git a/topics/design/understanding-saving-investing-and-speculating-key-differences-and-strategies.md b/topics/design/understanding-saving-investing-and-speculating-key-differences-and-strategies.md deleted file mode 100644 index 2ba947b2c..000000000 --- a/topics/design/understanding-saving-investing-and-speculating-key-differences-and-strategies.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: "Understanding saving, investing, and speculating: key differences and strategies" -description: "In personal finance, the strategies we choose to manage our money can significantly impact our financial future. These strategies typically fall into three categories: saving, investing, and speculating. Each comes with its own levels of risk, timeframes, and methods. Understanding the distinctions and appropriate uses of each can guide us in making smarter financial decisions..." -date: 2024-05-03 -authors: - - huytieu -tags: - - investment - - personal-finance -redirect: - - /mxK8aQ ---- - -In personal finance, the strategies we choose to manage our money can significantly impact our financial future. These strategies typically fall into three categories: saving, investing, and speculating. Each comes with its own levels of risk, timeframes, and methods. Understanding the distinctions and appropriate uses of each can guide us in making smarter financial decisions. - -## What Are saving, investing, and speculating? - -**Saving** involves putting money aside in secure forms like savings accounts or CDs. It’s characterized by very low risk and variable timeframes, suitable for short-term financial goals or as an emergency fund. -**Investing** is the process of purchasing assets, such as stocks, bonds, or real estate, with the expectation of long-term appreciation and profit. Investors take on moderate risks and typically aim for gradual wealth accumulation. -**Speculating**, on the other hand, involves taking significant risks for the potential of substantial, rapid returns. It’s often short-term and can involve high-volatility instruments like cryptocurrencies and options. - -## Key differences - -| Aspect | Saving | Investing | Speculating | -| --------- | ---------------- | -------------------- | ----------------- | -| Risk | Very low | Moderate | High | -| Timeframe | Variable | Long-term | Short-term | -| Approach | Regular deposits | Fundamental analysis | Market timing | -| Mindset | Security focus | Growth focus | Quick gains focus | - -## Strategies and risk management - -Effective financial management often involves utilizing all three strategies at different points in one’s life or simultaneously, depending on financial goals and risk tolerance. - -### Saving - -- **Goal**: Safeguard capital, ensuring funds are available when needed. -- **Method**: Regular contributions to savings vehicles. -- **Risk management**: Emphasizes capital preservation, low-risk instruments. - -### Investing - -- **Goal**: Accumulate wealth through appreciation and returns on capital. -- **Method**: Buy and hold assets, diversify across different investments. -- **Risk management**: Uses diversification to mitigate risks. - -### Speculating - -- **Goal**: Achieve significant profits in a short time frame. -- **Method**: Engage in frequent trades, often in high-risk markets. -- **Risk management**: Involves aggressive strategies, accepting the possibility of high losses for potential high returns. - -## Conclusions: practical advice - -Understanding your **risk tolerance** is crucial in choosing the right strategy. Thorough research is essential, especially for investing and speculating. Your **investment horizon**—whether you’re planning for the long term or looking for quick gains—should guide your choice of strategy. - -Diversification remains a fundamental principle of risk management across all types of financial strategies. -Remember, whether saving, investing, or speculating, tailor your strategies to align with your personal financial goals and risk appetite. Each method has its place in personal finance management, but they must be used wisely and with informed judgment. diff --git a/topics/design/use-jtbd-stories-to-foreground-the-context-in-which-a-solution-is-required.md b/topics/design/use-jtbd-stories-to-foreground-the-context-in-which-a-solution-is-required.md deleted file mode 100644 index 5b865ab25..000000000 --- a/topics/design/use-jtbd-stories-to-foreground-the-context-in-which-a-solution-is-required.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Use jtbd stories to foreground the context in which a solution is required -description: Learn how job stories focus on user context and motivations to improve product design, offering a clear alternative to traditional user stories by highlighting real user needs and situations. -date: null -redirect: - - /7ZoAEw ---- - -A "job story" is an alternative to the traditional user story format. In contrast to the user story, which foregrounds the persona performing an action, a job story highlights the context in which the action is performed. - -For example, where a user story may be written like this: - -> As a **working parent**, I want to **order take-out** so I can **spend time with my kids instead of cooking** - -A job story might be written like this: - -> **When I get home from a long day of work**, I want to **minimize the effort required to get a meal on the table** so I can **spend more time with my kids instead of cooking** - -More qualifiers can be added to increase the specificity of the job story, possibly promoting [empathy](https://publish.obsidian.md/mobydiction/notes/Different+forms+of+empathy): - -> When I get home from a long day of work, and **I’m tired** and **feel pressured** to put food on the table, I want to minimize the effort required to **prepare a healthy meal** so I can spend time with my kids instead of cooking. - -Job stories do not offer solutions; they describe the context for which the solution is hired. - ---- - -#### Related - -- [[Focus personas on tasks, goals, and motivations]] -- [[Personas make abstract research concrete for design teams]] -- [[Methods to overcome the limitations of personas]] -- [[JTBD Job Interview Structure]] - -#### Citations - -Kalbach, Jim, and Michael Schrage. The Jobs To Be Done Playbook: Align Your Markets, Organization, and Strategy Around Customer Needs. New York: Two Waves Books, 2020. - -Klement, Alan. “Replacing The User Story With The Job Story.” JTBD.Info (blog), July 3, 2018. [Link](https://jtbd.info/replacing-the-user-story-with-the-job-story-af7cdee10c27). diff --git a/topics/design/ux-model.md b/topics/design/ux-model.md deleted file mode 100644 index 2db5b3482..000000000 --- a/topics/design/ux-model.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Ux model -description: Good UX means creating useful, easy-to-use, and accessible products that attract users, build trust, and make information easy to find, ensuring a satisfying experience for all users. -date: 2018-03-19 -tags: - - design -redirect: - - /ikeBLw ---- - -## What is good UX? - -1. Useful - -A new product should first be designed to solve the users' problems. It should be the solution users expected. It is necessary to define the users' problem and search for ways to resolve it. Failure of defining the problem exactly will lead to failure of getting the right solution to users' expectation. Thus, the product will not be useful to them. - -2. Usability - -A product should be designed that it is easy to use. If it takes so much time for the users to get what they want out of the product, they would be likely to find another product with less struggles to deal with. - -3. Accessibility - -["Nearly 1 in 5 People Have a Disability in the U.S"](https://www.census.gov/newsroom/releases/archives/miscellaneous/cb12-134.html), so it is 20% of the users for any products you build targeting the US market. Designing a product with accessibility is to enable people with difficulty seeing, difficulty hearing, etc. to use your product. Considering about developing special versions of a product for those people is a good start. - -4. Desirability - -To get the users to try the product. You need to get their attention. There are many ways to attract users attention. The easiest way could be using well crafted visual designs for your products. Other ways are through branding, marketing, emotional design, interaction design, etc. - -5. Credibility - -To build a long relationship with your users, you need them to trust the product. Let them feel that your product is honest. And please be careful if you are building any features to collect private data of the users. It's not easy to get away with it. - -6. Findability - -There should be easy ways for users to search for information about the product. Something like the product name should be easy to remember, there are clear statements about your product you will want your users to remember, search, and talk about, etc. diff --git a/topics/design/what-i-learned-on-design-thinking-and-software-development.md b/topics/design/what-i-learned-on-design-thinking-and-software-development.md deleted file mode 100644 index a34c48824..000000000 --- a/topics/design/what-i-learned-on-design-thinking-and-software-development.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: What I learned on design thinking and software development -description: Learn how design thinking and the Software Development Life Cycle (SDLC) guide software projects to create user-focused products efficiently with quality, budget, and time management. -date: 2021-05-10 -tags: - - design-thinking -redirect: - - /UjlC1g ---- - -I'm Huyen Le - newcomer of Dwarves Foundation. Last week, I was taught under the instruction of Khai Le about design thinking and how it works in a design process of a software project. So I decided to write a recap of what I've learned. - -According to my knowledge, the basic definition of design thinking is an iterative process in which we understand the user's needs and problems. Define the problems to bring out the most ideal solutions, then we prototype the idea and test. - -### The five phases of design thinking - -- Empathize - Understand the audience. -- Define - Define the user's problems, needs and insight. -- Ideate - Create many ideas. Synthesize and select the most efficient ideas for innovative solutions. -- Prototype - Create solutions. -- Test - Test solutions. - -As a graphic designer and illustrator at the beginning, I realize that design thinking appeared in other fields like branding, event, art,... as well. The product development process in different fields seems to be the same, and all we want in a process is to guarantee the product is successful. - -Consequently, there is a methodology called Software Development Life Cycle (SDLC) that helps us produce software and possible. That means the software will be finished with the lowest budget and the highest quality in the shortest time. - -### The SDCL includes seven phases - -- Requirement analysis - Empathize, research, getting input -- Planning - Manage project constraints -- System design - Define, ideate, prototype. -- Implementation - Product development. An important stage to decide to project quality. -- Testing - Test and fix. -- Deployment - Release and use the product. -- Maintenance - Keep enhancing and optimizing the deployed product. - -Those phases are compulsory in software development, but I think the first 2 phases are essential. Because we need to know what we are going to build, its purpose and the current problem. More importantly, we must have a plan about the cost and the risk of the project if it fails. In the planning phase, three constraints that decide the success of a project are quality, budget and time. There are 2 common SDLC models: Waterfall and Agile. - -- **The Waterfall model** goes straight from the beginning to the ending. In this model, we finish one stage and then go to the next one. But this model has a restriction - *any small error in one phase can affect the whole process.* -- **The Agile model** has an iterative cycle that allows us to test and fix each outcome to make a new and better version. - -This is the first time I've gotten to know about the development software process properly. As a designer, I understand that creating a product needs more than a beautiful visual. It must be easy for users to use. And finally, the most important thing is: _”A good product is a product finished in the shortest time with the lowest cost and the highest quality”_. diff --git "a/topics/design/\302\266 Jobs to be done.md" "b/topics/design/\302\266 Jobs to be done.md" deleted file mode 100644 index e3d2a6426..000000000 --- "a/topics/design/\302\266 Jobs to be done.md" +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: ¶ jobs to be done -description: Learn how the Jobs to Be Done (JTBD) framework helps product teams focus on customer needs and outcomes by understanding the processes customers use to achieve their goals. -date: null -redirect: - - /xEeUHQ ---- - -Jobs to be done (JTBD) is a product development methodology that assumes that people only engage with products or services insofar as they help them satisfy some fundamental need, called a job. A "job" is a process of reaching some objective under given circumstances. It might be emotional, functional, or social. And, any given job may itself comprise many smaller jobs. - -Jobs tend to be stable over time, in contrast to solutions, which may change as new technologies emerge. - -These jobs exist independently of the means used to accomplish them. Companies, in essence, compete with one another by creating value through solving the job better than anyone else. - -According to Jim Kalbach, JTBD is less of a methodology and more a way of thinking. It helps organizations and teams deliver value by focusing on what customers actually want to achieve. It reminds product teams that, in general, customers don't want to actually interact with their company; they just want to arrive at their desired end state. - -In contrast to personas, which [tend to lack deep context](https://publish.obsidian.md/mobydiction/notes/Personas+lack+rich+context), JTBD brings context to the fore. - -There are five key points to remember: - -1. JTBD as a framework assumes that nobody actually wants your product. They want to achieve an outcome or objective. -2. A job is a process that someone undertakes to achieve that outcome. Your product can help with this. -3. Jobs can be multi-faceted. Main jobs can comprise many subordinate jobs. They can be functional, but also emotional or social. -4. Customer segments are best defined by shared needs and desired outcomes, not demographics. -5. JTBD does not specify solutions. It’s about capturing needs. - ---- - -#### Related - -- [[¶ Persona]] -- [[Methods to overcome the limitations of personas]] -- [[JTBD Job Interview Structure]] -- [[Use JTBD stories to foreground the context in which a solution is required]] -- [[Focus product development on customer jobs that are important, tangible, unsatisfied, and lucrative]] - -#### Citations - -Kalbach, Jim, and Michael Schrage. _The Jobs To Be Done Playbook: Align Your Markets, Organization, and Strategy Around Customer Needs_. New York: Two Waves Books, 2020. diff --git "a/topics/design/\302\266 Persona.md" "b/topics/design/\302\266 Persona.md" deleted file mode 100644 index 65a7a0f31..000000000 --- "a/topics/design/\302\266 Persona.md" +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: ¶ persona -description: User personas help product teams understand user needs, behaviors, and goals by creating realistic profiles that guide design and improve user experience in marketing and development. -date: null -redirect: - - /fEiH4w ---- - -User personas are research artifacts that describe patterns of needs, behaviour, and attributes of users. Usually presented as a character or a description of a specific, individual user, personas are commonly used to help product design teams understand and empathize with likely users. Personas draw attention to some aspects of the user and make concrete the abstract findings of research. - -Notably, different disciplines use personas in different ways. A persona developed and used by a marketing group, for instance, may describe a desired buyer of the product and include a great deal of demographic detail; user personas, on the other hand, tend to describe people who are likely to encounter the problem the product or design is meant to solve. Marketing personas and user personas may look similar, but they're not interchangeable. They do different jobs for different people. - -Personas may be broad or narrow in scope, depending on need. A broad persona will lack the rich detail necessary for focused design work, whereas a narrow personas will probably be too specific to influence larger-scale decision-making. But each have their place. - -A user persona should be realistic, not idealized, and serve to help teams understand the context in which a user might look to a product or design to help accomplish a goal or solve a problem. - ---- - -#### Related - -- [[Personas alone can't help teams achieve empathy]] -- [[Personas document shared understanding about users]] - -#### Citations - -Cooper, Alan. _The Inmates Are Running the Asylum_. Sams Publishing, 2004. - -Haag, Maren, and Nicola Marsden. _“Exploring Personas as a Method to Foster Empathy in Student IT Design Teams.”_ International Journal of Technology and Design Education, April 24, 2018. [Link](https://doi.org/10.1007/s10798-018-9452-5). - -Harley, Aurora. _“Personas Make Users Memorable for Product Team Members.”_ Nielsen Norman Group (blog). Accessed October 10, 2020. [Link](https://www.nngroup.com/articles/persona/). - -Salazar, Kim. _“Just-Right Personas: How to Choose the Scope of Your Personas.”_ Nielsen Norman Group. Accessed October 12, 2020. [Link](https://www.nngroup.com/articles/persona-scope/). - -Young, Indi. _“Describing Personas.”_ Medium (blog), August 13, 2020. [Link](https://medium.com/inclusive-software/describing-personas-af992e3fc527) diff --git a/topics/devops/about-devops.md b/topics/devops/about-devops.md deleted file mode 100644 index aef6c9490..000000000 --- a/topics/devops/about-devops.md +++ /dev/null @@ -1,157 +0,0 @@ ---- -title: About Devops -description: Learn how DevOps improves software delivery with continuous integration, automation, testing, and collaboration between development and operations for faster, reliable releases. -date: 2018-07-23 -tags: - - devops -redirect: - - /OvfD2A ---- - -## What is DevOps actually? - -DevOps is a culture with only one goal which is "improving the software lifecycle" (bug fixes, features, configurations) to end-users more frequently but still keep software's reliable. - -It is a continuous process and contains various stages such as : - -- Continuous Integration -- Continuous Development -- Continuous Testing -- Continuous Deployment -- Continuous Monitoring - -The main role of DevOps is to increase the quality of product to a great extent and to increase the collaboration of Development and Operation team as well so that the workflow within the organization becomes smoother. - -## Top core DevOps attributes - -- Ability to code and script (Go, Python, Shell script) -- Process re-engineering -- Communicating and collaborating with other -- Comfortable working with distributed teams -- Ability to use a wide variety of open source technologies and tools -- Networking / system admin skills -- Comfort with frequent, incremental code testing and deployment -- Strong grasp of automation tools -- Data management skills -- A strong focus on business outcomes - -Process re-engineering is probably the most telling skill. Engineers are not being hired to write code from scratch as much, but to find the best open source tools that can function with a company’s current platform and operating systems. - -## SRE table - -Availability is generally calculated based on how long a service was unavailable over some period. Assuming no planned downtime, Table 1-1 indicates how much downtime is permitted to reach a given availability level. - -![](assets/about-devops_9332ba5aa78b621a63f04a098e8ff602_md5.webp) - -Using an aggregate unavailability metric (i.e., "**_X_**% of all operations failed") is more useful than focusing on outage lengths for services that may be partially available—for instance, due to having multiple replicas, only some of which are unavailable—and for services whose load varies over the course of a day or week rather than remaining constant. - -See Equations [Time-based availability](https://landing.google.com/sre/book/chapters/embracing-risk.html#risk-management_measuring-service-risk_time-availability-equation) and [Aggregate availability](https://landing.google.com/sre/book/chapters/embracing-risk.html#risk-management_measuring-service-risk_aggregate-availability-equation) in [Embracing Risk](https://landing.google.com/sre/book/chapters/embracing-risk.html) for calculations. - -Reference: [https://landing.google.com/sre/book/chapters/availability-table.html](https://landing.google.com/sre/book/chapters/availability-table.html) - -## Role definitions - -### Software tester - -Goal of Automation Testing is to reduce number of test cases to be run manually and not eliminate Manual Testing all together. - -Types of software testing: - -- Unit Testing -- Functional Testing -- Regression Testing -- Black Box Testing -- Integration Testing -- Keyword Testing -- Data Driven Testing -- Smoke Testing - -The following category of test cases are not suitable for automation: - -- Test Cases that are newly designed and not executed manually atleast once -- Test Cases for which the requirements are changing frequently -- Test cases which are executed on ad-hoc basis. - -Automation Testing Tools: - -- Selenium -- QTP (MicroFocus UFT) -- Rational Functional Tester -- WATIR -- SilkTest - -### Security engineer - -Below is a list of the top five DevOps practices and tooling that can help improve overall security when incorporated directly into your end-to-end continuous integration/continuous delivery (CI/CD) pipeline: - -- Code Analysis with Continuous Code Quality (consistency, readability, performance, test coverage, vulnerabilities…) -- Security test automation -- Configuration and patch management -- Continuous monitoring - -Reference tools: - -- Code Quality: - - Codebeat - - Codacy - - SonarQube - - SonarLint for VS Code (Current support: JavaScript, PHP, Python, TypeScript) - - Codeclimate - - Codebeat - - gometalinter -- Monitoring: - - Sentry - - Datadog - -Deploying security solutions meeting one or more of the following security standards: NIST/FedRAMP, ISO 27001, ISO 27002, PCI DSS, **[HIPAA Security Rules](http://www.onlinetech.com/resources/references/what-is-the-hipaa-security-rule)** - -### Application developers - -Goal: a new software release must be deployed quickly. We need to uses processes as well as tools to streamline the software delivery process and reduce the overall cycle time. To help automate and integrate all of the essential delivery steps in a holistic way, the DevOps approach also needs lightweight tool changes. - -How do we do? - -- Think "automation" -- Write tests for continuous integration (CI). -- Sharing ideas, issues, processes, tools, and goals. - -Code and scripts for DevOps include the following: - -- Code and scripts for building the application. -- Code and scripts for unit testing the application. -- Code and scripts for acceptance testing the application. -- Code and scripts for deploying the application. -- Code and script configuration options for configuring the application for different target environments. -- Code and scripts for programming the attributes and “behavior” of the target environment. - -### System admin - -Goal: - -- Ability to apply their skills to entire IT infrastructures described and managed by code. -- Ability to manage cloud services and use automated deployment tools and code repositories - and to share their expertise with others. - -How to have Devops skills: - -- Needed: New technical skills -- Coding skill -- Cloud services: AWS, Azure, GCP, etc. -- Configuration Management and Infrastructure as code - -Reference tools: - -- Continuous integration servers: CircleCI, Gitlab runner, Jenkins, etc. -- Ansible, Puppet, Chef -- Orchestration tools: K8s, DC/OS, etc. - -## What will you do - -- Ability to automate. You should have an instinct and intuition to automate whatever you can and improve the efficiency of our development environments, processes. -- Coding skills. You don’t need to be a C++ wizard, or mastering in Go, but you should at least have a good background in one or more high level scripting language. -- Configuration management. You should have real world experience with something like Puppet, Chef, Saltstack, or Ansible. -- System / infrastructure / 3rd-party providers management. -- Documentation - -## Roadmap 2018 - -![](assets/about-devops_3ff5afb2faea481c0d85fe0d0f4591b5_md5.webp) diff --git a/topics/devops/assets/about-devops_3ff5afb2faea481c0d85fe0d0f4591b5_md5.webp b/topics/devops/assets/about-devops_3ff5afb2faea481c0d85fe0d0f4591b5_md5.webp deleted file mode 100644 index da936a071..000000000 Binary files a/topics/devops/assets/about-devops_3ff5afb2faea481c0d85fe0d0f4591b5_md5.webp and /dev/null differ diff --git a/topics/devops/assets/about-devops_9332ba5aa78b621a63f04a098e8ff602_md5.webp b/topics/devops/assets/about-devops_9332ba5aa78b621a63f04a098e8ff602_md5.webp deleted file mode 100644 index acc6377a2..000000000 Binary files a/topics/devops/assets/about-devops_9332ba5aa78b621a63f04a098e8ff602_md5.webp and /dev/null differ diff --git a/topics/devops/assets/blue-green-deployment-model.gif b/topics/devops/assets/blue-green-deployment-model.gif deleted file mode 100644 index 02189d211..000000000 Binary files a/topics/devops/assets/blue-green-deployment-model.gif and /dev/null differ diff --git a/topics/devops/assets/blue-green-deployment_bluegreen-blue-application.webp b/topics/devops/assets/blue-green-deployment_bluegreen-blue-application.webp deleted file mode 100644 index a13f87a66..000000000 Binary files a/topics/devops/assets/blue-green-deployment_bluegreen-blue-application.webp and /dev/null differ diff --git a/topics/devops/assets/blue-green-deployment_bluegreen-compare.webp b/topics/devops/assets/blue-green-deployment_bluegreen-compare.webp deleted file mode 100644 index 0f98a07ba..000000000 Binary files a/topics/devops/assets/blue-green-deployment_bluegreen-compare.webp and /dev/null differ diff --git a/topics/devops/assets/blue-green-deployment_bluegreen-green-application.webp b/topics/devops/assets/blue-green-deployment_bluegreen-green-application.webp deleted file mode 100644 index 39ed4b366..000000000 Binary files a/topics/devops/assets/blue-green-deployment_bluegreen-green-application.webp and /dev/null differ diff --git a/topics/devops/assets/deploy-branch-with-vercel-cli_action-comment-pr.webp b/topics/devops/assets/deploy-branch-with-vercel-cli_action-comment-pr.webp deleted file mode 100644 index 3040579a7..000000000 Binary files a/topics/devops/assets/deploy-branch-with-vercel-cli_action-comment-pr.webp and /dev/null differ diff --git a/topics/devops/assets/deploy-branch-with-vercel-cli_deploy-vercel-diagram.webp b/topics/devops/assets/deploy-branch-with-vercel-cli_deploy-vercel-diagram.webp deleted file mode 100644 index 72819aa0e..000000000 Binary files a/topics/devops/assets/deploy-branch-with-vercel-cli_deploy-vercel-diagram.webp and /dev/null differ diff --git a/topics/devops/assets/docker-microcontainers_6e719ca68d6f109ffe4061996ee21aa6_md5.webp b/topics/devops/assets/docker-microcontainers_6e719ca68d6f109ffe4061996ee21aa6_md5.webp deleted file mode 100644 index 838ebe6c3..000000000 Binary files a/topics/devops/assets/docker-microcontainers_6e719ca68d6f109ffe4061996ee21aa6_md5.webp and /dev/null differ diff --git a/topics/devops/assets/docker-microcontainers_9f15d2c4686cfe88873b99b117d8f372_md5.webp b/topics/devops/assets/docker-microcontainers_9f15d2c4686cfe88873b99b117d8f372_md5.webp deleted file mode 100644 index 1822d4201..000000000 Binary files a/topics/devops/assets/docker-microcontainers_9f15d2c4686cfe88873b99b117d8f372_md5.webp and /dev/null differ diff --git a/topics/devops/assets/docker-microcontainers_ae0b817a27464a31365a74e65dc37101_md5.webp b/topics/devops/assets/docker-microcontainers_ae0b817a27464a31365a74e65dc37101_md5.webp deleted file mode 100644 index c98e35c72..000000000 Binary files a/topics/devops/assets/docker-microcontainers_ae0b817a27464a31365a74e65dc37101_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-blue-green-deployment-helped-mochi_8a98dc8c92776e0f68f1db43bf3b4a9a_md5.webp b/topics/devops/assets/how-blue-green-deployment-helped-mochi_8a98dc8c92776e0f68f1db43bf3b4a9a_md5.webp deleted file mode 100644 index 74deac340..000000000 Binary files a/topics/devops/assets/how-blue-green-deployment-helped-mochi_8a98dc8c92776e0f68f1db43bf3b4a9a_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-blue-green-deployment-helped-mochi_8ee42dc2d4a9b07dbfc76db9b009a8cc_md5.webp b/topics/devops/assets/how-blue-green-deployment-helped-mochi_8ee42dc2d4a9b07dbfc76db9b009a8cc_md5.webp deleted file mode 100644 index f7d510522..000000000 Binary files a/topics/devops/assets/how-blue-green-deployment-helped-mochi_8ee42dc2d4a9b07dbfc76db9b009a8cc_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-blue-green-deployment-helped-mochi_dbebe30b5e9fa94fd865c1c66b41c5f6_md5.webp b/topics/devops/assets/how-blue-green-deployment-helped-mochi_dbebe30b5e9fa94fd865c1c66b41c5f6_md5.webp deleted file mode 100644 index 26570ae09..000000000 Binary files a/topics/devops/assets/how-blue-green-deployment-helped-mochi_dbebe30b5e9fa94fd865c1c66b41c5f6_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-blue-green-deployment-helped-mochi_f711c252bf191800d3184c8839221899_md5.webp b/topics/devops/assets/how-blue-green-deployment-helped-mochi_f711c252bf191800d3184c8839221899_md5.webp deleted file mode 100644 index 27db146f1..000000000 Binary files a/topics/devops/assets/how-blue-green-deployment-helped-mochi_f711c252bf191800d3184c8839221899_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-we-setup-cicd_9dd15aea3c3de09d92b44754a736b607_md5.webp b/topics/devops/assets/how-we-setup-cicd_9dd15aea3c3de09d92b44754a736b607_md5.webp deleted file mode 100644 index 88b1ca4fe..000000000 Binary files a/topics/devops/assets/how-we-setup-cicd_9dd15aea3c3de09d92b44754a736b607_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-we-setup-cicd_f38b955faad846c6c75b4252b56fe1b4_md5.webp b/topics/devops/assets/how-we-setup-cicd_f38b955faad846c6c75b4252b56fe1b4_md5.webp deleted file mode 100644 index 9443fbdd4..000000000 Binary files a/topics/devops/assets/how-we-setup-cicd_f38b955faad846c6c75b4252b56fe1b4_md5.webp and /dev/null differ diff --git a/topics/devops/assets/how-we-setup-cicd_fa32c5b22664bf943dd7d4314b012a83_md5.webp b/topics/devops/assets/how-we-setup-cicd_fa32c5b22664bf943dd7d4314b012a83_md5.webp deleted file mode 100644 index 694ff095b..000000000 Binary files a/topics/devops/assets/how-we-setup-cicd_fa32c5b22664bf943dd7d4314b012a83_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_17c2731e7fc3be306d1da3317bbee37a_md5.webp b/topics/devops/assets/kubernetes-helm-101_17c2731e7fc3be306d1da3317bbee37a_md5.webp deleted file mode 100644 index 4f9408223..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_17c2731e7fc3be306d1da3317bbee37a_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_426db2b83abcad55d8b8ce46b11fa2d0_md5.webp b/topics/devops/assets/kubernetes-helm-101_426db2b83abcad55d8b8ce46b11fa2d0_md5.webp deleted file mode 100644 index fdf078f17..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_426db2b83abcad55d8b8ce46b11fa2d0_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_63fcd3d5115887bdd2e46fd0f8c337b7_md5.webp b/topics/devops/assets/kubernetes-helm-101_63fcd3d5115887bdd2e46fd0f8c337b7_md5.webp deleted file mode 100644 index bdae35c93..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_63fcd3d5115887bdd2e46fd0f8c337b7_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_6a409c81c3372fda59fd6217eab2fabc_md5.webp b/topics/devops/assets/kubernetes-helm-101_6a409c81c3372fda59fd6217eab2fabc_md5.webp deleted file mode 100644 index 1395ef683..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_6a409c81c3372fda59fd6217eab2fabc_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_cf8bf24ef2e60c2dbc46df36ee6a9639_md5.webp b/topics/devops/assets/kubernetes-helm-101_cf8bf24ef2e60c2dbc46df36ee6a9639_md5.webp deleted file mode 100644 index 01af52224..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_cf8bf24ef2e60c2dbc46df36ee6a9639_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_ea686ec580638794529f8054ef266a45_md5.webp b/topics/devops/assets/kubernetes-helm-101_ea686ec580638794529f8054ef266a45_md5.webp deleted file mode 100644 index a62784e66..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_ea686ec580638794529f8054ef266a45_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubernetes-helm-101_f864eb297f5f2ff0a55adc3876a07a3c_md5.webp b/topics/devops/assets/kubernetes-helm-101_f864eb297f5f2ff0a55adc3876a07a3c_md5.webp deleted file mode 100644 index f5b6a0a5e..000000000 Binary files a/topics/devops/assets/kubernetes-helm-101_f864eb297f5f2ff0a55adc3876a07a3c_md5.webp and /dev/null differ diff --git a/topics/devops/assets/kubeseal-&-sops_kubeseal_&_sops.webp b/topics/devops/assets/kubeseal-&-sops_kubeseal_&_sops.webp deleted file mode 100644 index 037fdab8d..000000000 Binary files a/topics/devops/assets/kubeseal-&-sops_kubeseal_&_sops.webp and /dev/null differ diff --git a/topics/devops/assets/progressive-delivery_engineering-progressive-delivery.webp b/topics/devops/assets/progressive-delivery_engineering-progressive-delivery.webp deleted file mode 100644 index 88499b7aa..000000000 Binary files a/topics/devops/assets/progressive-delivery_engineering-progressive-delivery.webp and /dev/null differ diff --git a/topics/devops/assets/radio-talk-60-blue-green-deployment_3e12057cf9cee4df856d0720a11e0fc7_md5.gif b/topics/devops/assets/radio-talk-60-blue-green-deployment_3e12057cf9cee4df856d0720a11e0fc7_md5.gif deleted file mode 100644 index 02189d211..000000000 Binary files a/topics/devops/assets/radio-talk-60-blue-green-deployment_3e12057cf9cee4df856d0720a11e0fc7_md5.gif and /dev/null differ diff --git a/topics/devops/blue-green-deployment.md b/topics/devops/blue-green-deployment.md deleted file mode 100644 index c6e5bc323..000000000 --- a/topics/devops/blue-green-deployment.md +++ /dev/null @@ -1,220 +0,0 @@ ---- -title: Blue green deployment -description: Learn how to implement blue-green deployment in Kubernetes using Argo Rollouts to reduce downtime, improve reliability, and smoothly switch between production environments. -date: 2022-02-16 -github_id: nguyennh4522 -icy: 10 -redirect: - - /NI0LLA ---- - -**Blue-green** deployment is a software deployment strategy that involves creating two identical environments: one **blue** environment serving production traffic, and another **green** environment that doesn't serve any traffic. Once the **green** environment is fully tested and verified, traffic is switched from the **blue** environment to the **green** environment, making it the new production environment. This approach reduces downtime, improves reliability and resilience, and provides a backup in case of issues. - -![](assets/blue-green-deployment-model.gif) - -## How does blue-green deployment work? - -**Blue-green** deployment is a deployment strategy for software applications that involves maintaining two identical environments: one currently serving production traffic (the **blue** environment), and one that is newly deployed (the **green** environment"). The new version of the application is deployed to the **green** environment, which is tested and monitored. Once it is determined that the **green** environment is working correctly, traffic is routed to it and the **blue** environment is retired. This strategy allows for quick and easy switching between environments, minimizing downtime and reducing the risk of errors or bugs. - -## Characteristic - -The following table summarizes the salient features of the **blue-green** strategy compared to other strategies: - -![](assets/blue-green-deployment_bluegreen-compare.webp) - -## Implementing blue-green deployment strategy in Kubernetes - -1. Preparing - -Before implementing blue-green deployment in Kubernetes, there are a few things you should do to prepare: - -- Set up a Kubernetes cluster with nginx-ingress controller, cert-manager, argo-rollouts -- Domain for active application and preview application -- Define application resources: - ``` - . - └── app/ - ├── bluegreen-rollout.yaml - ├── ingress.yaml - └── service.yaml - ``` - -2. Implementation - -To implement blue-green deployment in Kubernetes, you need create `bluegreen-rollout.yaml` file to defines the application resources and the rollout strategy: - -```yaml -# bluegreen-rollout.yaml -apiVersion: argoproj.io/v1alpha1 -kind: Rollout -metadata: - name: myapp - labels: - app: myapp -spec: - replicas: 1 - revisionHistoryLimit: 1 - selector: - matchLabels: - app: myapp - template: - metadata: - labels: - app: myapp - spec: - containers: - - name: myapp - image: argoproj/rollouts-demo:green - imagePullPolicy: Always - ports: - - name: http - containerPort: 8080 - strategy: - blueGreen: - autoPromotionEnabled: false - activeService: myapp - previewService: myapp-preview -``` - -`bluegreen-rollout.yaml` defined like a normal deployment file. The only difference is the `strategy` section: - -- `autoPromotionEnabled`: will make the rollout automatically promote the new ReplicaSet to the active service once the new ReplicaSet is healthy. This field is defaulted to true if it is not specified (default: `true`). -- `activeService`: specifies the service to update with the new template hash at time of promotion. This field is required. -- `previewService`: specifies the service to update with the new template hash before promotion. This field is optional. - -Next, you need to create `ingress.yaml` file and `service.yaml` file to define the ingress and service resources for the application: - -```yaml -# ingress.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: myapp - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: nginx - tls: - - hosts: - - myapp.bluegreen.xyz - secretName: myapp.bluegreen.xyz-tls - rules: - - host: myapp.bluegreen.xyz - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: myapp - port: - name: http - ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: myapp-preview - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - ingressClassName: nginx - tls: - - hosts: - - myapp-preview.bluegreen.xyz - secretName: myapp-preview.bluegreen.xyz-tls - rules: - - host: myapp-preview.bluegreen.xyz - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: myapp-preview - port: - name: http -``` - -```yaml -# service.yaml -apiVersion: v1 -kind: Service -metadata: - name: myapp - labels: - app: myapp -spec: - selector: - app: myapp - ports: - - name: http - port: 80 - targetPort: http - type: ClusterIP - ---- -apiVersion: v1 -kind: Service -metadata: - name: myapp-preview - labels: - app: myapp -spec: - selector: - app: myapp - ports: - - name: http - port: 80 - targetPort: http - type: ClusterIP -``` - -Apply these resources to the cluster: - -```bash -kubectl apply -f app/bluegreen-rollout.yaml -kubectl apply -f app/service.yaml -kubectl apply -f app/ingress.yaml -``` - -Now, application is deployed on active environment and you can view it on `myapp.bluegreen.xyz` domain. - -![](assets/blue-green-deployment_bluegreen-green-application.webp) - -We change the image of the application to `argoproj/rollouts-demo:blue` and apply the changes to the cluster: - -```yaml -# bluegreen-rollout.yaml ---- -containers: - - name: myapp - image: argoproj/rollouts-demo:blue -``` - -```bash -kubectl apply -f app/bluegreen-rollout.yaml -``` - -Argo rollouts will create a new ReplicaSet with the new image and start to rollout the new version of the application to the preview environment. You can view the preview application on `myapp-preview.bluegreen.xyz` domain. - -![](assets/blue-green-deployment_bluegreen-blue-application.webp) - -When the new version of the application is ready, you can promote it to the active environment by running the following command: - -```bash -kubectl argo rollouts promote myapp -``` - -Now, the application is deployed on active environment and you can view it on `myapp.bluegreen.xyz` domain. - -## Conclusion - -Implementing blue-green deployment in Kubernetes requires preparation, including setting up a Kubernetes cluster, containerizing the application, defining application resources in manifests, setting up a CI/CD pipeline, and implementing monitoring and logging. However, once set up, blue-green deployment can help increase the reliability, availability, and quality of the deployed application. By minimizing downtime, reducing risks, and ensuring the smooth transition to new versions of the application, blue-green deployment can ultimately help organizations deliver more value to their customers. - -## References - -- https://www.redhat.com/en/topics/devops/what-is-blue-green-deployment -- https://cloud.google.com/architecture/application-deployment-and-testing-strategies -- https://argoproj.github.io/argo-rollouts/ -- https://viblo.asia/p/kubernetes-practice-english-automating-bluegreen-deployment-with-argo-rollouts-GAWVpoGaL05 diff --git a/topics/devops/docker-microcontainers.md b/topics/devops/docker-microcontainers.md deleted file mode 100644 index 456590505..000000000 --- a/topics/devops/docker-microcontainers.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: Docker microcontainers -description: Learn how to create ultra-small Docker images using the scratch image to deploy Go web apps with minimal size and faster downloads by cross-compiling and building lightweight containers. -date: 2016-02-04 -tags: - - docker -redirect: - - /6SlwUg ---- - -When using Docker, you will quickly realize that the image using to run a project takes a big capacity. A simple image ubuntu took nearly 200MB, but you don’t take full advantage of its available tools. Other images like Node, Go, etc almost run in Ubuntu, debian and only set up more environments to easily deploy. However, downloading a new image takes more time because of its big capacity and it seems to be not necessary at all. - -I found an empty image, except some added metadata by Docker - -```javascript -docker pull scratch -``` - -Therefore, this Docker image has the smallest capacity - -## How to deploy an app web with this Docker image - -- I will demo with beego web. -- I cannot compile app go in this empty image because it has no Go complier. And while using Mac, I cannot compile a Linux library. In reality, we absolutely can cross-compile but I will not mention in this blog. You can refer to [here](https://golang.org/doc/install/source#environment) -- So I need to create an environment to compile and docker build source Go - -```javascript -docker run --rm -it -v "$GOPATH":/go -e "GOPATH=/go" -v /var/run/docker.sock:/var/run/docker.sock -v $(which docker):$(which docker) golang bash -``` - -- With the above command, `-rm` will delete container after exit,`v "$GOPATH":/go` share directory GOPATH to container, `e "GOPATH=/go", -v /var/run/docker.sock:/var/run/docker.sock` is used to run Docker into container (docker in docker). `v $(which docker):$(which docker)` is used to share command docker, golang is image environment to build Go binary and build image, bash to begin Bash session. -- This time, because I went into container, I cd to project `/go/src/git.dwarvesf.com/ivkean/web.` -- Create a Dockerfile with the content below - -```javascript -# scratch -FROM scratch - -WORKDIR /web - -# copy binary into image -COPY web web - -# copy other necessary files -COPY conf conf -COPY static static -COPY views views - -EXPOSE 8080 - -ENTRYPOINT ["/web/web"] -``` - -- Build source by command: `go build` -- And link library by:`CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o web .` -- Build with docker: `docker build -t beego-web .` - -![](assets/docker-microcontainers_9f15d2c4686cfe88873b99b117d8f372_md5.webp) - -- Creating an image only takes 17.5MB - -![](assets/docker-microcontainers_6e719ca68d6f109ffe4061996ee21aa6_md5.webp) - -- Your laptop had an image, which is beego-web - -![](assets/docker-microcontainers_ae0b817a27464a31365a74e65dc37101_md5.webp) diff --git a/topics/devops/feature-flags.md b/topics/devops/feature-flags.md deleted file mode 100644 index 4d1879b42..000000000 --- a/topics/devops/feature-flags.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Feature flags -description: Feature flags let software teams enable or disable features during runtime, supporting beta testing, A/B experiments, and smoother releases without multiple code branches or redeployments. -date: 2022-09-25 -github_id: zlatanpham -redirect: - - /fEZEkQ ---- - -Feature flags are a software engineering technique that allows you to enable or disable select functionality during runtime without having to deploy new code. The ability to control the visibility of features enables development teams to manage the full lifecycle of a feature — rather than being constrained by the traditional "code-deploy-test" cycle. - -By using feature flags, you eliminate the need to maintain multiple branches for different features in your source code. All your code changes can be made to the primary branch and then enabled via a flag when the new feature is ready. This is an essential practice to ensure that [[Progressive Delivery]] is done correctly. Feature Flags encourage trunk-based development by having developers commit code to a single branch (trunk) rather than long-lived feature or development branches. A single branch of code helps to eliminate merge conflicts, broken builds, and results in a cleaner codebase. Instead of using a feature branch, use a flag to gate features not ready for public viewing. - -Feature flags are a valuable tool for software teams, but they can also be leveraged by other teams within the organization. For example, help sales and customer team support employees to provision entitlements; enable product managers to manage beta programs; or allow marketing teams to run A/B tests. - -## Core benefits of feature flags - -- **Release management**: beta-test a new feature with a select group of users before releasing it to everyone. -- **Operation management**: monitor systems, toggling features on and off to minimize the impact of incidents. -- **Learn from experiment**: greatly support the use of A/B testing to test hypotheses and bring the best ideas to our customers. -- **Provision entitlements**: give a select group of users (normally, "premium" users) an early opportunity to try out a feature. - -Feature flags add value to an organization as a whole, not just to the development team. - -## Reference - -- https://launchdarkly.com/blog/what-are-feature-flags/ -- https://martinfowler.com/articles/feature-toggles.html diff --git a/topics/devops/kubernetes-helm-101.md b/topics/devops/kubernetes-helm-101.md deleted file mode 100644 index 9d8c22082..000000000 --- a/topics/devops/kubernetes-helm-101.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -title: Kubernetes Helm 101 -description: Learn how Helm, the Kubernetes package manager and deployment tool, simplifies application deployment with charts, releases, and repositories for faster, easier Kubernetes management. -date: 2019-10-30 -tags: - - k8s -redirect: - - /mCIe6w ---- - -In recent years, Kubernetes has exploded tremendously. At the same time, it creates new communities including ecosystems that make it easier to develop and deploy complex applications. If Kubernetes means the shipman of the ship, then Helm is the steering wheel of that shipman in the career of every DevOps and Developer. - -**Helm is a Package Manager for Kubernetes**, analogous to NPM or YARN. However, it’s not just the Package Manager, it is also a Deployment Management for Kubernetes. In simpler terms, instead of having to define various Kubernetes resources to deploy an application, with Helm you just type a few commands in the terminal and enter, done. - -![](assets/kubernetes-helm-101_6a409c81c3372fda59fd6217eab2fabc_md5.webp) - -In this article, we will learn about Helm, a powerful tool that makes life working with Kubernetes easier than ever. Although these terms are quite common with developers today, we will skim a jiffy for those who don’t know: - -- **Containerization**: _a next-generation virtualization and automation solution after Hypervisor Virtualization_. It widely applied today because of the breakthrough effect with outstanding advantages in creating and deploying applications faster, more scalable and more securely than traditional methods. -- **Docker**: _a number one software to create, manage and run containers._ Most chosen for all popular _Containerization_ applications today. Many large cloud solutions like Google Cloud, Amazon Web Services, Microsoft Azure are using it. -- **Kubernetes**: _a famous open source used for container orchestration_. With the rise of _Containerization_, the management and coordination of container-based applications became complicated and difficult, making Kubernetes an effective and indispensable solution for systems using Containerization - -## Helm overview - -### Concepts - -Helm has 4 basic concepts: - -- **Chart**: a collection of YAML files; bundle of the Kubernetes resources needed to build a Kubernetes application. For ease of visualization, Helm Chart can be compared like a Docker Image. Of course, Helm also has a Helm Hub where to search and share Charts for popular apps - -![](assets/kubernetes-helm-101_6a409c81c3372fda59fd6217eab2fabc_md5.webp) - -- **Config**: a configuration in the values.yaml file, which contains configuration explicit to a release of Kubernetes application. It can be the config for service, ingress, deployment, etc. until specific applications such as Kafka, Consul, Vault, NATS-streaming, etc. - -![](assets/kubernetes-helm-101_ea686ec580638794529f8054ef266a45_md5.webp) - -- **Release**: a chart instance is loaded into Kubernetes. It can be viewed as a version of the Kubernetes application running based on Chart and associated with a specific Config. - -![](assets/kubernetes-helm-101_17c2731e7fc3be306d1da3317bbee37a_md5.webp) - -- **Repositories**: a repository of published Charts. These can be private repositories that are only used within the company or public through the Helm Hub. Some Charts may have different versions of many companies or publishers. Particularly, Charts in a Stable repository must always meet the criteria from the Technical Requirements of Helm. - -![](assets/kubernetes-helm-101_426db2b83abcad55d8b8ce46b11fa2d0_md5.webp) - -## Architecture - -Helm has a fairly simple client-server architecture, including a CLI client and an in-cluster server running in the Kubernetes cluster: - -![](assets/kubernetes-helm-101_63fcd3d5115887bdd2e46fd0f8c337b7_md5.webp) - -- **Helm client**: Provides the developer to use it a command-line interface (CLI) to work with Charts, Config, Release, Repositories. Helm client will interact with Tiller server, to perform various actions such as _install_, _upgrade_ and _rollback_ with Charts, Release. -- **Tiller server**: an in-cluster server in the Kubernetes cluster, interacting with the Helm client and communicating with the Kubernetes API server. Thus, Helm can easily manage Kubernetes with tasks such as _install_, _upgrade_, _query_ and _remove_ for Kubernetes resources. - -Here are some basic concepts and architectures to help you understand and grasp Helm more quickly. In the next section, we’ll take some tutorials to deep dive into Helm. Let’s do it! - -## Helm in action - -Before we begin, we’ll need to prepare a few things for us to be able to practice with Helm. - -- Firstly, we need a Kubernetes cluster. At [Dwarves Foundation](https://dwarves.foundation/), we’re using the Google Kubernetes Engine for a production environment and the Kubernetes on-premise for a staging environment. If you don’t have any Kubernetes cluster, don’t worry. You can use [Minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/), which offers an excellent way to work with a single-node Kubernetes cluster locally. -- And, we’ll need a basic purpose that manages the application within the Kubernetes cluster. In fact, I have a responsibility for overseeing and configuring [Sol](https://solapp.io/), the Travel Log product of the [Dwarves Foundation](https://dwarves.foundation/). It using Kafka as a message streaming for communication between its’ microservices. For this tutorial, we’ll install Kafka to the Kubernetes cluster via Helm. - -## **Installing Helm** - -There are multiple ways to install Helm that are obviously described on the [Helm documentation](https://helm.sh/docs/using_helm/#installing-helm). Helm has two parts: The Helm client (helm) and the Helm server (Tiller). **The quickest way to install helm on macOS is using** **[Homebrew](https://brew.sh/)**, a package manager for macOS platforms. - -It's a simple command to install Helm client locally via Homebrew - -```javascript -brew install kubernetes-helm -``` - -Next step is to initialize helm, please ensure that the Kubernetes cluster is running and accessible through _kubectl_. When you initialize helm, a deployment named tiller-deploy will be deployed in the kube-system namespace. - -Initialize helm using the following command. - -```javascript -helm init -``` - -Check the tiller deployment in the kube-systems namespace using **_kubectl _**command - -```javascript -kubectl get deployment tiller-deploy -n kube-system -``` - -## Deploy an application using Helm - -Now we’re going to deploy a Kafka cluster using helm. - -It’s many charts of Kafka when we search on the Helm Hub. Unfortunately, Kafka doesn’t have a Stable chart, so you can choose which one fits with your use case. For this tutorial, we’ll install the Kafka chart of Bitnami into queue-production namespace. - -As a simple way, we can install Kafka with helm by the following command - -```javascript -# Add Incubator repository -helm repo add bitnami https://charts.bitnami.com/bitnami - -# Update helm repository -helm repo update - -# Install Kafka -helm install bitnami/kafka --name kafka --namespace queue -``` - -Now, it already installed on your Kubernetes cluster, but it may not work properly for your needs. For me, I usually looking for a values.yaml file, and store it locally to save a specific configuration for instance of the application. - -![](assets/kubernetes-helm-101_cf8bf24ef2e60c2dbc46df36ee6a9639_md5.webp) - -Execute the following helm install command to deploy a Kafka with your Config in `values-prod.yaml`. It will download the Kafka helm chart from the Bitnami repo and apply your configuration via `values.yaml file`. - -```javascript -helm install bitnami/kafka \ - --name kafka-prod \ - --namespace queue-production \ - -f bitnami/values-prod.yaml -``` - -![](assets/kubernetes-helm-101_f864eb297f5f2ff0a55adc3876a07a3c_md5.webp) - -Check the install helm chart using this command - -```javascript -helm ls -``` - -Delete the installation from Kubernetes cluster, use this command - -```javascript -helm delete --purge kafka-prod -``` - -## Conclusion - -We’ve witnessed the ecosystem around Kubernetes blossom, and new tools are appearing every day. Helm is an essential tool for DevOps and Developer using Kubernetes in their production environment. Tools like Helm are often used when considering quick deployment strategies and cost savings in operations. - -You can refer to more information in [the documentation of Helm](https://helm.sh/docs/). In the next article, I will show you how to create a Helm Chart of NATS-streaming. Hopefully, this article will help you who are intending to learn about Helm as well as the necessary tools of the DevOps. Thank you, please give me your commend if there are deficiencies and I will do better in the next articles. - -### Read more - -- [What is containerization?](https://hackernoon.com/what-is-containerization-83ae53a709a6#targetText=Containerization%20involves%20bundling%20an%20application,ecosystems%20are%20Docker%20and%20Kubernetes.) -- [Helm documentation](https://helm.sh/docs/) -- [Using Helm and Kubernetes](https://www.baeldung.com/kubernetes-helm) -- [What is Helm and why is it important for Kubernetes deployments?](https://boxboat.com/2018/09/19/helm-and-kubernetes-deployments/) -- [Helm tutorial: how to install and configure Helm](https://devopscube.com/install-configure-helm-kubernetes/) diff --git a/topics/devops/kubeseal-sops.md b/topics/devops/kubeseal-sops.md deleted file mode 100644 index 41cd7f056..000000000 --- a/topics/devops/kubeseal-sops.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Kubeseal sops -description: Learn how to securely manage Kubernetes secrets using Kubeseal and Sops to encrypt and store secrets safely in public repos while ensuring only your cluster can decrypt them. -date: 2022-09-28 -github_id: nguyennh4522 -redirect: - - /Bri4uw ---- - -- `Kubeseal`: Encrypt your Secret into a SealedSecret, which is safe to store - even to a public repository. The SealedSecret can be decrypted only by the controller running in the target cluster and nobody else (not even the original author) is able to obtain the original Secret from the SealedSecret. SealedSecret is composed of two parts: - - - Cluster-side controller: `sealed-secrets-controller` - generate keypair and decrypt secrect. - - Client-side utility: `kubeseal` - uses asymmetric crypto to encrypt secrets that only the controller can decrypt. - -- `Sops`: Sops is a binary able to encrypt configuration files. But rather than encrypting the whole file Sops understands format (JSON, YAML, INI, etc) and will only encrypt the values of each line (with AWS KMS, GCP KMS, Azure Key Vault, age, and PGP). - -## How it works - -![](assets/kubeseal-&-sops_kubeseal_&_sops.webp) - -- Source code repo: contains secret was encrypted by `sops` -- Github action: use `sops` to decrypt secret and `kubeseal` to seal secret and push it to infrastructure repo -- Infrastructure repo: save all Kubernetes config -- Cluster: use `sealed-secrets-controller` to decrypt sealedsecret to secret and apply it - -## Core benefits - -- **Easy management**: Developers can update secret without devops. -- **Security**: The secret will be encrypted and cannot be decrypted without the key provided by devops. - -## Reference - -- https://github.com/bitnami-labs/sealed-secrets -- https://github.com/mozilla/sops diff --git a/topics/devops/progressive-delivery.md b/topics/devops/progressive-delivery.md deleted file mode 100644 index 987d12748..000000000 --- a/topics/devops/progressive-delivery.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Progressive delivery -description: Progressive delivery lets teams release new software features gradually using feature flags and mature CI/CD pipelines to reduce risks and improve user experience. -date: 2022-09-25 -github_id: zlatanpham -redirect: - - /FdF4Xg ---- - -Progressive delivery is a modern software development methodology for gradually rolling out new features in order to assess the user response and limit the potential negative impact. With progressive delivery, features are released first to an internal QA team, then to real users in a controlled, measured manner. - -One of the key advantages of progressive delivery is that it makes your release process more resilient to errors and negative user experiences. For instance, if you detect errors at 1% of your traffic, you've only potentially impacted 1% of revenue or 1% of customer satisfaction. - -![](assets/progressive-delivery_engineering-progressive-delivery.webp) - -## Progressive delivery contract - -The practice of progressive delivery sounds good in theory but requires strict requirements to make it effective: - -- CI/CD is already part of your delivery pipeline -- `master` branch is always releasable -- Every new feature includes a [[Feature flags | feature flag]] - -### Mature CI/CD pipelines - -When starting out with progressive delivery, teams need mature CI/CD pipelines. A mature process includes: - -- Automated build and automated testing triggered for every commit. -- Developers should commit their changes frequently into the baseline. -- Deployment can be accomplished with a simple command or click. - -### Master is always releasable - -Every change merged into `master` must preserve the releasability of `master`. Releasable means the revision can be released to production with the requirements of: - -- Changelog and documentation reflect the current feature set. -- All features have undergone quality assurance by a set of appropriate users in an appropriate environment. -- The regression test suite passes. - -### A feature flag is required for every new feature - -Feature flags are one of the core components of progressive delivery. Without them, it is impossible to test in production on real users with very low risk. Feature flags achieve 2 things: - -- They allow continuous release to be maintained while still allowing big features to be merged into main (disabled by a feature flag) before they are fully tested. -- They give us a quick way to disable features in the event that a feature breaks the release. - -## Reference - -- https://www.optimizely.com/optimization-glossary/progressive-delivery/ -- https://handbook.sourcegraph.com/departments/engineering/dev/tools/continuous_releasability/ -- https://www.cloudbees.com/blog/progressive-delivery-vs-continuous-delivery diff --git a/topics/devops/testing-aws-services-locally-with-localstack.md b/topics/devops/testing-aws-services-locally-with-localstack.md deleted file mode 100644 index 3da3a27c3..000000000 --- a/topics/devops/testing-aws-services-locally-with-localstack.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -title: Testing aws services locally with localstack -description: LocalStack lets developers locally test and develop AWS services like S3, Lambda, and DynamoDB without AWS costs, speeding up cloud application development with a simple, cost-free environment. -date: 2023-03-17 -github_id: nguyend-nam -icy: 10 -redirect: - - /13TOGA ---- - -**Amazon Web Services** (AWS) provides a wide range of cloud computing services that can be used to build and deploy applications at scale. However, using AWS can be costly, especially for small-scale projects or for developers who are just starting out. That's where [LocalStack](https://localstack.cloud/) comes in. LocalStack is an open-source tool that enables **local development** and **testing** of cloud applications by emulating various AWS services on a developer's local side. It can be used to test and develop applications **without** incurring the cost and complexity of using live AWS environment. - -## Terminologies - -**AWS** is a flexible, scalable, and cost-effective cloud computing platform that offers a wide range of services to help individuals, startups, and enterprises build and manage their applications and infrastructure. - -> Learn more about [AWS](https://aws.amazon.com/). - -**LocalStack** supports many AWS services, including S3, Kinesis, and DynamoDB, etc., and provides a simple API for interacting with them. It is available for use in many programming languages or customize the environment. With LocalStack, you can test and develop your applications without incurring any costs, making it an ideal solution for developers who want to learn and experiment with AWS services without committing to a full-scale deployment. - -> Download Python and follow [LocalStack's documentation](https://docs.localstack.cloud/getting-started/installation/) to complete installation, registration and add LocalStack's API key to the environment. - -## Example - -AWS **S3** (Simple Storage Service) is designed to store and retrieve large amounts of data, including structured and unstructured data, in buckets, which are logical containers for objects (files) that can be accessed using the S3 API, SDKs, or the AWS Management Console. - -In this brainery post, let's take an example of LocalStack AWS S3 to demonstrate how we can test an AWS service locally without needing to connect to a live AWS environment. - -```bash -pip3 install awscli-local -``` - -Firstly, run the command above to be able to use the **`awslocal`** command. Detailed information can be found at [LocalStack AWS CLI (awslocal)](https://docs.localstack.cloud/user-guide/integrations/aws-cli/#localstack-aws-cli-awslocal). - -Then start docker and LocalStack, run the command below to **create a bucket** named `sample-bucket`: - -```bash -awslocal s3api create-bucket --bucket sample-bucket -``` - -```bash -{ - "Location": "/sample-bucket" -} -``` - -To **view the buckets list**, run: - -```bash -awslocal s3api list-buckets -``` - -```bash -{ - "Buckets": [ - { - "Name": "sample-bucket", - "CreationDate": "2023-03-15T04:32:52+00:00" - } - ], - "Owner": { - "DisplayName": "webfile", - "ID": "bcaf1ffd86f41161ca5fb16fd081034f" - } -} -``` - -Next, let's **store** an html file to the bucket we just created. Create an `index.html` with some random content at the current location and run: - -```bash -awslocal s3api put-object --bucket sample-bucket --key index.html --body index.html -``` - -```bash -{ - "ETag": "\"d73997cf9bba06462b3ebe94c3743b2e\"" -} -``` - -```bash -awslocal s3api get-object --bucket sample-bucket --key index.html output.txt -``` - -```bash -{ - "AcceptRanges": "bytes", - "LastModified": "2023-03-15T07:00:36+00:00", - "ContentLength": 266, - "ETag": "\"d73997cf9bba06462b3ebe94c3743b2e\"", - "VersionId": "null", - "ContentLanguage": "en-US", - "ContentType": "binary/octet-stream", - "Metadata": {} -} -``` - -The command above is used for exporting the content of the file we just stored to an `output.txt` file. - -You can also test other operations on an AWS S3 bucket locally with LocalStack. Let say you want to **set up replication** to objects from one S3 bucket to another for backup and recovery purposes, then you need to run the command: - -```bash -awslocal s3api put-bucket-replication --bucket sample-bucket --replication-configuration '{"Role": "arn:aws:iam::123456789012:role/replication-role","Rules": [{"Status": "Enabled","Priority": 1,"DeleteMarkerReplication": {"Status": "Disabled"},"Destination": {"Bucket": "arn:aws:s3:::sample-replica-bucket","AccessControlTranslation": {"Owner": "Destination"}}}]}' -``` - -Note that you'll need to provide real values for `Role` and `Bucket` when working with AWS S3, right here we just need dummy values to test with LocalStack. To **view the replication**, run: - -```bash -awslocal s3api get-bucket-replication --bucket sample-bucket -``` - -```bash -{ - "ReplicationConfiguration": { - "Role": "arn:aws:iam::123456789012:role/replication-role", - "Rules": [ - { - "Priority": 1, - "Status": "Enabled", - "Destination": { - "Bucket": "arn:aws:s3:::sample-replica-bucket", - "AccessControlTranslation": { - "Owner": "Destination" - } - }, - "DeleteMarkerReplication": { - "Status": "Disabled" - } - } - ] - } -} -``` - -## Other services - -Some of the emulation services for other AWS APIs that LocalStack supports: - -- [Elastic compute cloud (EC2)](https://docs.localstack.cloud/user-guide/aws/elastic-compute-cloud/) - -- [CloudFront](https://docs.localstack.cloud/user-guide/aws/cloudfront/) - -- [Lambda](https://docs.localstack.cloud/user-guide/aws/lambda/) - -- [Kinesis](https://docs.localstack.cloud/user-guide/aws/kinesis/) - -- [DynamoDB](https://docs.localstack.cloud/user-guide/aws/dynamodb/) - -> See the [Service Feature Coverage](https://docs.localstack.cloud/user-guide/aws/feature-coverage/) on LocalStack's documentation for further information of the AWS APIs that LocalStack has covered. - -## Benefits - -There are several advantages of using LocalStack over the original AWS solutions: - -1. Cost-saving: LocalStack is free to use and doesn't incur any AWS usage costs. - -2. Faster development: Since LocalStack is a local development environment, we don't need to deploy code to a remote environment for testing, hence save time and increase development speed. - -3. Better control: LocalStack provides developers with greater control over their testing environment. Developers can create custom test scenarios and modify the environment as needed, without affecting any live AWS environments. - -4. Improved accuracy: LocalStack's local environment allows developers to test their code more accurately, as they can simulate real-world scenarios without the risk of impacting live data or users. - -## Reference - -- https://docs.localstack.cloud/user-guide/aws/s3/ -- https://aws.amazon.com/s3/?nc2=h_ql_prod_st_s3 diff --git a/topics/elixir/Garbage collector in Elixir.md b/topics/elixir/Garbage collector in Elixir.md deleted file mode 100644 index 262fefbf5..000000000 --- a/topics/elixir/Garbage collector in Elixir.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: null -description: null -date: null -tags: - - elixir - - garbage -redirect: - - /6pkPXQ ---- - -BEAM process has its own stack and heap for small data binaries(<64bytes) Larger binaries are stored in a shared memory space with a ref-counted pointer(called ProcBin) that lives in a Process's heap. There are many data stacks and heaps in our application, one per process. - -- Generational - minor garbage collection. This method reclaims **heap memory** that is no longer referenced by the process. Idea: most memory is only used for a short period of time. It is **highly performant**, but looks at **recently allocated** - -- Full-sweep - major garbage collection. This method looks at the entire heap of a process, and reclaims as **much memory as possible**. Idea: memory must be clean when the heap is close to being full. It is fast in practice, but it can be slow if executing on a process that is actively holding a lot of data - -### How to collect data manually - -1. From OTP 20 provide a function to reclaims the memory in a process: :erlang.garbage_collect/1. We can use :erlang.garbage_collect/0 to trigger garbage collection for the process that invoked the function -2. Passing the :hibernate_after option for GenServer.start/3 function. The data will be reclaims when the process is idled. Ex: Phoenix Channel use hibernate_after to enter hibernation 15 seconds after processing their last message. -3. Returning :hibernate at the end of the GenServer callbacks - -```elixir -def handle_call(:clear, _from, _state) do - {:reply, :ok, [], :hibernate} -end -``` diff --git a/topics/elixir/erlang-fsm.md b/topics/elixir/erlang-fsm.md deleted file mode 100644 index e3b8a9807..000000000 --- a/topics/elixir/erlang-fsm.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Erlang finite state machine -description: The Power of Finite State Machines -date: 2024-07-12 -authors: - - hieuphq -tags: - - elixir - - erlang - - fsm -redirect: - - /pa7seA ---- - -Finite State Machines (FSMs) are a crucial concept in computer science and software development, providing a robust method for modeling the behavior of systems. Erlang, a language designed for concurrency, fault tolerance, and distributed computing, offers unique advantages when implementing FSMs. In this note, we'll explore how Erlang excels in handling FSMs, using the Catch Chicken Machine as a practical example. - -## Why Erlang? - -1. **Concurrency**: Erlang's lightweight processes and efficient message-passing make it ideal for concurrent systems. -2. **Fault tolerance**: Built-in mechanisms for error detection and recovery enhance system reliability. -3. **Distributed computing**: Native support for distributed systems allows easy scaling and resilience. - -## The catch chicken machine: a practical example - -The Catch Chicken Machine is a simple yet illustrative example of a finite state machine implemented in Erlang. It models a system where a chicken-catching robot moves between states based on external inputs and internal logic. - -## States and transitions - -- **Idle**: The initial state, waiting for a command to start. -- **Searching**: Actively looking for a chicken. -- **Catching**: Attempting to catch a detected chicken. -- **Resting**: Taking a break after a catch or a failed attempt. - -## Implementation practices - -1. **Define states and events**: Clearly define each state and the events that trigger transitions. - - ```erlang - -define(STATES, [idle, searching, catching, resting]). - -define(EVENTS, [start, chicken_spotted, chicken_caught, chicken_escaped, rest]). - ``` - -2. **State transition logic**: Use Erlang's pattern matching to implement transition logic. - - ```erlang - handle_event(start, idle) -> - {next_state, searching}; - handle_event(chicken_spotted, searching) -> - {next_state, catching}; - handle_event(chicken_caught, catching) -> - {next_state, resting}; - handle_event(chicken_escaped, catching) -> - {next_state, searching}; - handle_event(rest, resting) -> - {next_state, idle}. - ``` - -3. **Concurrency and messaging**: Leverage Erlang's messaging capabilities to handle state transitions. - - ```erlang - loop(State) -> - receive - Event -> - {next_state, NewState} = handle_event(Event, State), - loop(NewState) - end. - ``` - -4. **Fault tolerance**: Implement error handling to ensure the FSM can recover from unexpected states or failures. - ```erlang - handle_event(_, _) -> - {next_state, idle}. - ``` - -## Benefits of using Erlang for FSMs - -- **Scalability**: Erlang's ability to handle numerous concurrent processes allows the FSM to scale efficiently. -- **Reliability**: The language's emphasis on fault tolerance ensures that the FSM can recover gracefully from errors. -- **Clarity**: Pattern matching and clear state definitions make the FSM logic easy to understand and maintain. - -## Conclusion - -Erlang's unique features make it an excellent choice for implementing finite state machines. By following best practices and leveraging the language's strengths, developers can create robust, scalable, and reliable FSMs. The Catch Chicken Machine example demonstrates how Erlang's concurrency, fault tolerance, and clear syntax contribute to effective FSM implementation. Embrace Erlang for your FSM needs and experience the benefits firsthand. diff --git a/topics/engineering/202210122014-forward-proxy.md b/topics/engineering/202210122014-forward-proxy.md deleted file mode 100644 index dccd2227d..000000000 --- a/topics/engineering/202210122014-forward-proxy.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: Forward proxy -description: A forward proxy is a server that sits between user devices and the internet. -date: 2022-10-12 -authors: - - trankhacvy -discord_id: "khacvy#8287" -tags: - - engineering - - forward-proxy -redirect: - - /pFVrbg ---- - -**Forward proxy**: is a server that sits between user devices and the internet. -uses for: - -- protect clients -- avoid browsing restrictions -- block access to certain content - -**Reverse proxy:** is a server that accepts a request from the client, forwards the request to web servers, and returns the results to the client as if the proxy server had processed the request. -uses for: - -- Protect servers -- load balancing -- cache static contents -- encrypt and decrypt SSL communications - -Credit: https://blog.bytebytego.com/p/ep25-proxy-vs-reverse-proxy - -![](assets/202210122014---forward-proxy_pasted-image-20221012202942.webp) diff --git a/topics/engineering/202210131000-behavior-driven-development.md b/topics/engineering/202210131000-behavior-driven-development.md deleted file mode 100644 index 5a9fa23fe..000000000 --- a/topics/engineering/202210131000-behavior-driven-development.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Behavior driven development -description: Behavior driven development (BDD) is a software development process that encourages collaboration among developers, QA, and non-technical stakeholders. -date: 2022-10-13 -authors: - - huytieu -discord_channel: engineering -tags: - - behavior-driven-development - - agile - - testing - - ubiquitous-language -redirect: - - /bCvFdw ---- - -### Behavior driven development (BDD) Three Practices - -- First, take a small upcoming change to the system – a User Story – and talk about concrete examples of the new functionality to explore, discover and agree on the details of what’s expected to be done. -- Next, document those examples in a way that can be automated, and check for agreement. -- Finally, implement the behaviour described by each documented example, starting with an automated test to guide the development of the code. - The aim is to: -- make small changes rapidly -- back up 1 level when need more info -- automate and implement a new example to get feedback - BDD Example: - -``` -- Feature: User Login -*As a user I want to login into the Company's website using my existing account so that I can use other features* -- Scenario Outline: Login with valid credential -Given I navigate to -When I input -and I input -and I click Login Button -Then I should be able to login successfully -- Examples: -| Email | Password -| abc@company.info | password 1 -| abc2@company.info | Password@ -``` - -> Personal note on implementing BDD and TDD: -> -> - Mistakes are that people only implement BDD/TDD practices at **Testing phase** when they implemented automation test. However it should be as early as possible. Using "BDD" to only cover automation test is not **BDD**. -> - BDD helps products and developers realize different scenarios of a same feature, then cover as much as they could at requirement and implementation phase. -> - BDD borrows _ubiquitous language_ concept from domain driven design so that everyone in the team can understand the term the same way. - -Source: diff --git a/topics/engineering/202211081111-error-messaging.md b/topics/engineering/202211081111-error-messaging.md deleted file mode 100644 index 819a835b4..000000000 --- a/topics/engineering/202211081111-error-messaging.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Error messaging -description: What makes a bad error message -date: 2022-11-08 -authors: - - trankhacvy -tags: - - ux -redirect: - - /Jcdkug ---- - -What makes a bad error message - -- Inappropriate tone -- Technical jargon -- Passing the blame -- Generic for no reason - -What makes a good error message - -- Say what happened and why -- Provide reassurance -- Be empathetic -- Help them fix it -- Always give a way out - -Reference: - -![](assets/202211081111---error-messaging_pasted-image-20221108111328.webp) -![](assets/202211081111---error-messaging_pasted-image-20221108111346.webp) diff --git a/topics/engineering/a-and-cname-records-in-dns.md b/topics/engineering/a-and-cname-records-in-dns.md deleted file mode 100644 index d8664205c..000000000 --- a/topics/engineering/a-and-cname-records-in-dns.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: A and cname records in dns -description: Learn the key differences between A records and CNAME records in DNS, including their uses for domain mapping, subdomains, and IP address management to optimize your website setup. -date: 2022-01-06 -github_id: namtrhg -icy: 10 -redirect: - - /71LEhQ ---- - -**A record** and **CNAME** are one of the most common types of records when you want to deploy your `domain` or `subdomain` on the internet. - -## Use-cases and restrictions - -### A record - -`A` record is an abbreviation for "Address". The address you type when you go to a website, send an email or connect to Twitter, Facebook or Instagram. -There are many things you can do with `A` records, including using multiple `A` records for the same domain to offer redundancy and fallbacks. Furthermore, many names may point to the same IP address, in which case each will have its own `A` record pointing to the same IP address. - -![](assets/a-and-cname-records-in-dns_a_record_config_picture.webp) - -#### Use-cases - -- Use an `A` record if you manage which IP addresses are assigned to a particular machine, or if the IP are fixed (this is the most common case). - -### CNAME record - -`A CNAME` is a database entry in the Domain Name System (DNS) that indicates that one domain name is a alias for another. The `CNAME`, often known as the "true name" is especially crucial when multiple services are running from the same IP address. - -![](assets/a-and-cname-records-in-dns_cname_record_config_picture.webp) - -#### Use-cases - -- To direct people to the main website from many websites owned by the same individual or organization. -- To assign a unique hostname to each network service, such as File Transfer Protocol (FTP) or email, and point it to the root domain. -- To assign a `subdomain` to each customer on the domain of a single service provider and use `CNAME` to point the `subdomain` to the customer's `root domain`. -- To register the same domain in many countries and direct each country's version to the main domain. - -#### Restrictions - -- A `CNAME` record should always link to another domain name rather than an IP address. -- A `CNAME` record cannot exist alongside another record with the same name. It is not feasible for `www.example.com` to have both a 'CNAME' and a 'TXT' record. -- A `CNAME` can point to another CNAME. However, this is not normally advised for performance reasons. To prevent needless performance overheads, the `CNAME` should point as nearly as feasible to the destination name when relevant. - -## References - -- -- -- -- . diff --git a/topics/engineering/architecture-decision-record.md b/topics/engineering/architecture-decision-record.md deleted file mode 100644 index 85f08b8a6..000000000 --- a/topics/engineering/architecture-decision-record.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: Architecture decision record -description: Learn how Architecture Decision Records (ADR) help teams document software choices, improve project clarity, and guide newcomers with clear context, solutions, and consequences for better long-term development. -date: 2020-03-27 -tags: - - architecture - - engineering -redirect: - - /loTgRA ---- - -There is an issue that every team will inevitably bump into. Newbies might get confused when a project is being developed. They wonder how that code was written, how we ended up choosing that architecture. It somehow leads to finger-point, and even the decision-maker himself can't remember how that decision was taken. - -That happens in a long-term project, and that's fine. We believe we're not the only one who gets that, so we'd like to gather you all and tackle this today by a practice called Architecture decision record (ADR) - -## So the main list item would be - -- Why ADR? -- What it contains -- Demo of ADR and example -- Support tools for ADR - -![](assets/architecture-decision-record_4d815bb79330db7c753064e343bbf411_md5.webp) - -So first, the picture demonstrated a crew of soldiers in a room, seeking for bugs to destroy. Once the bugs were found and the boss asked his soldiers to put an end on it, they told him not to touch anything, because the whole room would collapse. And they all ran out. So no bugs were ended. - -It matches our story. When we start to write up a system, no matter how small, if we don't log the decision at the time we make it, we won't even be understanding later then, much less, to decide whether or not to adjust it. It stops us from improving the system. - -When a newbie joins, they can't just immediately catch up with the current status. They don't understand any of that. Likely, newbies tend to accept the current decision without digging into the root cause. Another question would be: What happens if we break the current system and start all over again? To avoid that, they build up a practice called _ADR, to log the info and context that comes along with the decision of architecture or a technical decision_. It's like a doc of history. - -Take the currency as an example. Most people don't get why the government switched from coin to paper money. Because we didn't live at the time that decision was made, and if there was nothing as 'history log,' we won't understand the reason behind and accept it as an improved move back then. - -ADR helps provide a context of architecture, explaining why the previous PIC decided to make a move on it, with two main focuses: Context and Consequences - -## A kit of ADR - -- Architecture decision (AD): A software design choice to adjust the architecture. -- Architecture decision log (ADL): A series of files that logs our decision through different versions. ADL helps to remind what the previous decision was to make the next one better -- Requirement: the necessary condition for that architecture - -## How to write an ADR - -1. First, the decision must be brought up. This part is called AD, and that comes with two questions - -- with the current system, does this decision crucial and matter enough? -- does this need to be done immediately? → should we make the decision - -2. Decision making -> Finalized the decision -3. Implement that decision with the system and have all the related-parties aware The decision must be agreed by many parties (stakeholder, business, design, dev) - -## Note down the ADR - -ADR can be in different ways. After full observation, we have selected the most simple yet combined mutual from all of them. In general, an ADR is made out of three components: - -1. Context: the environment and situation that leads to the decision, the current business requirement, the problem, and the constraint. -2. Solution: the selected option that outweighs the others, explain its pros and cons -3. Consequences: the impact of that decision, describe how that decision change the system or any change log to note down - -## Good ADR - -- Point in Time: must be stated clearly -- Rationality: explain the reason behind that decision -- Immutable Record: the decision is finalized and cannot be altered -- Specificity: and ADR should only be about _one_ decision only - -## Good context - -Provide the ADR with the current system and business context. -This helps drive the broader view and the business situation at that time. - -## Good consequences - -The right approach explains the result from making that decision and how it adjusts the current business status. - -## Template of an ADR - -### Alexandrian pattern - -- Prologue (a summary) -- Discussion (Context) -- Solution -- Consequences - -### How to manage with Git - -[https://github.com/npryce/adr-tools/](https://github.com/npryce/adr-tools/) - -We also had some discussion afterward, demos, and Q&A sessions. But that was a brief intro of how an ADR practice should be adopted. diff --git a/topics/engineering/assets/202210122014---forward-proxy_pasted-image-20221012202942.webp b/topics/engineering/assets/202210122014---forward-proxy_pasted-image-20221012202942.webp deleted file mode 100644 index e284b476e..000000000 Binary files a/topics/engineering/assets/202210122014---forward-proxy_pasted-image-20221012202942.webp and /dev/null differ diff --git a/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111328.webp b/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111328.webp deleted file mode 100644 index aa80d5560..000000000 Binary files a/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111328.webp and /dev/null differ diff --git a/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111346.webp b/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111346.webp deleted file mode 100644 index cc391c7bd..000000000 Binary files a/topics/engineering/assets/202211081111---error-messaging_pasted-image-20221108111346.webp and /dev/null differ diff --git a/topics/engineering/assets/a-and-cname-records-in-dns_a_record_config_picture.webp b/topics/engineering/assets/a-and-cname-records-in-dns_a_record_config_picture.webp deleted file mode 100644 index 82b194945..000000000 Binary files a/topics/engineering/assets/a-and-cname-records-in-dns_a_record_config_picture.webp and /dev/null differ diff --git a/topics/engineering/assets/a-and-cname-records-in-dns_cname_record_config_picture.webp b/topics/engineering/assets/a-and-cname-records-in-dns_cname_record_config_picture.webp deleted file mode 100644 index 65363309c..000000000 Binary files a/topics/engineering/assets/a-and-cname-records-in-dns_cname_record_config_picture.webp and /dev/null differ diff --git a/topics/engineering/assets/analysis-document_analysis-doc-sample-first.webp b/topics/engineering/assets/analysis-document_analysis-doc-sample-first.webp deleted file mode 100644 index 75e1d8135..000000000 Binary files a/topics/engineering/assets/analysis-document_analysis-doc-sample-first.webp and /dev/null differ diff --git a/topics/engineering/assets/analysis-document_analysis-doc-sample-second.webp b/topics/engineering/assets/analysis-document_analysis-doc-sample-second.webp deleted file mode 100644 index c3f5cc712..000000000 Binary files a/topics/engineering/assets/analysis-document_analysis-doc-sample-second.webp and /dev/null differ diff --git a/topics/engineering/assets/analysis-document_analysis-doc-sample-third.webp b/topics/engineering/assets/analysis-document_analysis-doc-sample-third.webp deleted file mode 100644 index dd5f1476c..000000000 Binary files a/topics/engineering/assets/analysis-document_analysis-doc-sample-third.webp and /dev/null differ diff --git a/topics/engineering/assets/architecture-decision-record_4d815bb79330db7c753064e343bbf411_md5.webp b/topics/engineering/assets/architecture-decision-record_4d815bb79330db7c753064e343bbf411_md5.webp deleted file mode 100644 index 243506ffa..000000000 Binary files a/topics/engineering/assets/architecture-decision-record_4d815bb79330db7c753064e343bbf411_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/are-we-really-engineers_22ca6a7a31c29b15355a09a6cbd7129b_md5.webp b/topics/engineering/assets/are-we-really-engineers_22ca6a7a31c29b15355a09a6cbd7129b_md5.webp deleted file mode 100644 index 0ca276c58..000000000 Binary files a/topics/engineering/assets/are-we-really-engineers_22ca6a7a31c29b15355a09a6cbd7129b_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/are-we-really-engineers_6cf3675a02ca90a8745a69479b359671_md5.webp b/topics/engineering/assets/are-we-really-engineers_6cf3675a02ca90a8745a69479b359671_md5.webp deleted file mode 100644 index 3d372a1dc..000000000 Binary files a/topics/engineering/assets/are-we-really-engineers_6cf3675a02ca90a8745a69479b359671_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/are-we-really-engineers_cfc74d29d5032e4015de69984f910062_md5.webp b/topics/engineering/assets/are-we-really-engineers_cfc74d29d5032e4015de69984f910062_md5.webp deleted file mode 100644 index 3f79d4a86..000000000 Binary files a/topics/engineering/assets/are-we-really-engineers_cfc74d29d5032e4015de69984f910062_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/build-an-assistant-on-the-terminal_898d0616614b4483301cd1793967a1ef_md5.webp b/topics/engineering/assets/build-an-assistant-on-the-terminal_898d0616614b4483301cd1793967a1ef_md5.webp deleted file mode 100644 index 85c3e366d..000000000 Binary files a/topics/engineering/assets/build-an-assistant-on-the-terminal_898d0616614b4483301cd1793967a1ef_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/burnup-and-burndown-chart_burndown_chart_example.webp b/topics/engineering/assets/burnup-and-burndown-chart_burndown_chart_example.webp deleted file mode 100644 index c9aa0c2a9..000000000 Binary files a/topics/engineering/assets/burnup-and-burndown-chart_burndown_chart_example.webp and /dev/null differ diff --git a/topics/engineering/assets/burnup-and-burndown-chart_burnup_chart.webp b/topics/engineering/assets/burnup-and-burndown-chart_burnup_chart.webp deleted file mode 100644 index a8a9b8e3b..000000000 Binary files a/topics/engineering/assets/burnup-and-burndown-chart_burnup_chart.webp and /dev/null differ diff --git a/topics/engineering/assets/daemons-and-services-programming-guide_6b7ebc46edfa8c99dbdbfb540f15bb11_md5.webp b/topics/engineering/assets/daemons-and-services-programming-guide_6b7ebc46edfa8c99dbdbfb540f15bb11_md5.webp deleted file mode 100644 index 337c99334..000000000 Binary files a/topics/engineering/assets/daemons-and-services-programming-guide_6b7ebc46edfa8c99dbdbfb540f15bb11_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/daemons-and-services-programming-guide_8f72ec64cac909319e7fc79210c69563_md5.webp b/topics/engineering/assets/daemons-and-services-programming-guide_8f72ec64cac909319e7fc79210c69563_md5.webp deleted file mode 100644 index 20d1dcddb..000000000 Binary files a/topics/engineering/assets/daemons-and-services-programming-guide_8f72ec64cac909319e7fc79210c69563_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/daemons-and-services-programming-guide_93dcafb893273d168521685e97d42ad8_md5.webp b/topics/engineering/assets/daemons-and-services-programming-guide_93dcafb893273d168521685e97d42ad8_md5.webp deleted file mode 100644 index 089d8f154..000000000 Binary files a/topics/engineering/assets/daemons-and-services-programming-guide_93dcafb893273d168521685e97d42ad8_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/data-first-approach-with-zod-00.jpg b/topics/engineering/assets/data-first-approach-with-zod-00.jpg deleted file mode 100644 index 99367362b..000000000 Binary files a/topics/engineering/assets/data-first-approach-with-zod-00.jpg and /dev/null differ diff --git a/topics/engineering/assets/data-first-approach-with-zod-01.jpg b/topics/engineering/assets/data-first-approach-with-zod-01.jpg deleted file mode 100644 index 7366deefa..000000000 Binary files a/topics/engineering/assets/data-first-approach-with-zod-01.jpg and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_04501b291da585ce90758a6b363be5e9_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_04501b291da585ce90758a6b363be5e9_md5.webp deleted file mode 100644 index 87396cfaf..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_04501b291da585ce90758a6b363be5e9_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_41a7d2c490ccbd641b6c819e39fefdeb_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_41a7d2c490ccbd641b6c819e39fefdeb_md5.webp deleted file mode 100644 index 142f19e71..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_41a7d2c490ccbd641b6c819e39fefdeb_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_5a3973c10e77842fa50e038b4c9755b8_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_5a3973c10e77842fa50e038b4c9755b8_md5.webp deleted file mode 100644 index 077328b4c..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_5a3973c10e77842fa50e038b4c9755b8_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_6b6695453f90a53f40ccc71a42275e36_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_6b6695453f90a53f40ccc71a42275e36_md5.webp deleted file mode 100644 index 55274e047..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_6b6695453f90a53f40ccc71a42275e36_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_c9a1b2c55d33bd432d3b0ace8d0a65e7_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_c9a1b2c55d33bd432d3b0ace8d0a65e7_md5.webp deleted file mode 100644 index 403eaee0a..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_c9a1b2c55d33bd432d3b0ace8d0a65e7_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/database-designs-for-multilingual-apps_f257e0952b3c5b44d18722936afa96b6_md5.webp b/topics/engineering/assets/database-designs-for-multilingual-apps_f257e0952b3c5b44d18722936afa96b6_md5.webp deleted file mode 100644 index f967d7aff..000000000 Binary files a/topics/engineering/assets/database-designs-for-multilingual-apps_f257e0952b3c5b44d18722936afa96b6_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-1---quick-look-installation_7e7988a963f67f1005ed0e19e2b93e01_md5.webp b/topics/engineering/assets/dcos-series-part-1---quick-look-installation_7e7988a963f67f1005ed0e19e2b93e01_md5.webp deleted file mode 100644 index 636c2d258..000000000 Binary files a/topics/engineering/assets/dcos-series-part-1---quick-look-installation_7e7988a963f67f1005ed0e19e2b93e01_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_377366f7ddcabc286a3d6d1e505eb94f_md5.webp b/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_377366f7ddcabc286a3d6d1e505eb94f_md5.webp deleted file mode 100644 index 0e29fe6a4..000000000 Binary files a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_377366f7ddcabc286a3d6d1e505eb94f_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_82cc4eeeaae867eb9bcafc54c0f7eda5_md5.webp b/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_82cc4eeeaae867eb9bcafc54c0f7eda5_md5.webp deleted file mode 100644 index f31132c60..000000000 Binary files a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_82cc4eeeaae867eb9bcafc54c0f7eda5_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_8f79be9b29c62d95fe90e2e03bc62284_md5.webp b/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_8f79be9b29c62d95fe90e2e03bc62284_md5.webp deleted file mode 100644 index cc6bfe5a0..000000000 Binary files a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_8f79be9b29c62d95fe90e2e03bc62284_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_e84b0a38362cbe07380b58ee7adae02d_md5.webp b/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_e84b0a38362cbe07380b58ee7adae02d_md5.webp deleted file mode 100644 index ceb1eed04..000000000 Binary files a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_e84b0a38362cbe07380b58ee7adae02d_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_fad27ec33399ea4f93224edcc53bbfb3_md5.webp b/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_fad27ec33399ea4f93224edcc53bbfb3_md5.webp deleted file mode 100644 index 1d8946f75..000000000 Binary files a/topics/engineering/assets/dcos-series-part-2---deploy-simple-applications_fad27ec33399ea4f93224edcc53bbfb3_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_93c6fdb9cee529cbeb2a4141d910bb8f_md5.webp b/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_93c6fdb9cee529cbeb2a4141d910bb8f_md5.webp deleted file mode 100644 index 9c0c26b0d..000000000 Binary files a/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_93c6fdb9cee529cbeb2a4141d910bb8f_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_d90d63eed205052f3f560b4a6ebd5fba_md5.webp b/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_d90d63eed205052f3f560b4a6ebd5fba_md5.webp deleted file mode 100644 index 0ef86542b..000000000 Binary files a/topics/engineering/assets/dcos-series-part-3---service-discovery-and-load-balancing_d90d63eed205052f3f560b4a6ebd5fba_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_07ae10e7b5195c92905e0901562cb533_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_07ae10e7b5195c92905e0901562cb533_md5.webp deleted file mode 100644 index f755fb8ee..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_07ae10e7b5195c92905e0901562cb533_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_3d03a4c6a74681114944d4f0cc0ca683_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_3d03a4c6a74681114944d4f0cc0ca683_md5.webp deleted file mode 100644 index b98e42cf0..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_3d03a4c6a74681114944d4f0cc0ca683_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_7274a46eb1cff2f39783f79ef6352709_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_7274a46eb1cff2f39783f79ef6352709_md5.webp deleted file mode 100644 index c8a2d1b20..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_7274a46eb1cff2f39783f79ef6352709_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_91935fc991d145d9dec0c0a5a1703c2e_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_91935fc991d145d9dec0c0a5a1703c2e_md5.webp deleted file mode 100644 index 410fe4d6f..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_91935fc991d145d9dec0c0a5a1703c2e_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b85d3535f4be0d3605d8d61ef3fcd21a_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b85d3535f4be0d3605d8d61ef3fcd21a_md5.webp deleted file mode 100644 index 29e6a9f7f..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b85d3535f4be0d3605d8d61ef3fcd21a_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b987a78c85ca24c80127c39227dc905f_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b987a78c85ca24c80127c39227dc905f_md5.webp deleted file mode 100644 index 6db2396f0..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_b987a78c85ca24c80127c39227dc905f_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_bd74914f8224427cee07a129310fbe15_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_bd74914f8224427cee07a129310fbe15_md5.webp deleted file mode 100644 index 12e5e7d50..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_bd74914f8224427cee07a129310fbe15_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_d235654ef8f0f428f5ecad332edb6148_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_d235654ef8f0f428f5ecad332edb6148_md5.webp deleted file mode 100644 index d69cf7284..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_d235654ef8f0f428f5ecad332edb6148_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_efe59623d7bf4e6f6112c0d8b5fc2385_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_efe59623d7bf4e6f6112c0d8b5fc2385_md5.webp deleted file mode 100644 index 288e8e7aa..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_efe59623d7bf4e6f6112c0d8b5fc2385_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_fb4564d4758af4cc151fca604136180f_md5.webp b/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_fb4564d4758af4cc151fca604136180f_md5.webp deleted file mode 100644 index 74ab57513..000000000 Binary files a/topics/engineering/assets/dcos-series-part-4---deploy-simple-application-with-backend-database_fb4564d4758af4cc151fca604136180f_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-5---gitlab_07cb48f22769c4bdb7555e122aa2c24c_md5.webp b/topics/engineering/assets/dcos-series-part-5---gitlab_07cb48f22769c4bdb7555e122aa2c24c_md5.webp deleted file mode 100644 index d22ee7573..000000000 Binary files a/topics/engineering/assets/dcos-series-part-5---gitlab_07cb48f22769c4bdb7555e122aa2c24c_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-5---gitlab_0a331d4343f65dc0f3d0a1d28fcd08a6_md5.webp b/topics/engineering/assets/dcos-series-part-5---gitlab_0a331d4343f65dc0f3d0a1d28fcd08a6_md5.webp deleted file mode 100644 index fcc63bcad..000000000 Binary files a/topics/engineering/assets/dcos-series-part-5---gitlab_0a331d4343f65dc0f3d0a1d28fcd08a6_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-5---gitlab_1293c7d7f3ddd11d274b6a78461f93a5_md5.webp b/topics/engineering/assets/dcos-series-part-5---gitlab_1293c7d7f3ddd11d274b6a78461f93a5_md5.webp deleted file mode 100644 index 4087c955b..000000000 Binary files a/topics/engineering/assets/dcos-series-part-5---gitlab_1293c7d7f3ddd11d274b6a78461f93a5_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-5---gitlab_4a171771af9dd3725d809a02f3cbd80a_md5.webp b/topics/engineering/assets/dcos-series-part-5---gitlab_4a171771af9dd3725d809a02f3cbd80a_md5.webp deleted file mode 100644 index 7ee41b11a..000000000 Binary files a/topics/engineering/assets/dcos-series-part-5---gitlab_4a171771af9dd3725d809a02f3cbd80a_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/dcos-series-part-5---gitlab_6d0a8c8c9d3f0ea69d31fd97da4de9b1_md5.webp b/topics/engineering/assets/dcos-series-part-5---gitlab_6d0a8c8c9d3f0ea69d31fd97da4de9b1_md5.webp deleted file mode 100644 index e8d48bf14..000000000 Binary files a/topics/engineering/assets/dcos-series-part-5---gitlab_6d0a8c8c9d3f0ea69d31fd97da4de9b1_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_backend-architecture-with-graphql-gateway-example.webp b/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_backend-architecture-with-graphql-gateway-example.webp deleted file mode 100644 index 1f693f173..000000000 Binary files a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_backend-architecture-with-graphql-gateway-example.webp and /dev/null differ diff --git a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_graphql-schema-federation-example.webp b/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_graphql-schema-federation-example.webp deleted file mode 100644 index 83520e3f9..000000000 Binary files a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_graphql-schema-federation-example.webp and /dev/null differ diff --git a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_legacy-old-architecture-example.webp b/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_legacy-old-architecture-example.webp deleted file mode 100644 index 674354604..000000000 Binary files a/topics/engineering/assets/graphql-in-microservices---unified-api-gateway_legacy-old-architecture-example.webp and /dev/null differ diff --git a/topics/engineering/assets/how-to-run-the-backlog-grooming-effectively_backlog_grooming_note_example.webp b/topics/engineering/assets/how-to-run-the-backlog-grooming-effectively_backlog_grooming_note_example.webp deleted file mode 100644 index eb0e5069a..000000000 Binary files a/topics/engineering/assets/how-to-run-the-backlog-grooming-effectively_backlog_grooming_note_example.webp and /dev/null differ diff --git a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_2e152cf173f2ed991e018bb6126f6cc3_md5.webp b/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_2e152cf173f2ed991e018bb6126f6cc3_md5.webp deleted file mode 100644 index 9a7e65502..000000000 Binary files a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_2e152cf173f2ed991e018bb6126f6cc3_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_541015267939c46a3258073ebd192e01_md5.webp b/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_541015267939c46a3258073ebd192e01_md5.webp deleted file mode 100644 index ecb37825d..000000000 Binary files a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_541015267939c46a3258073ebd192e01_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8ae015f43c500413e1239f24be2847cd_md5.webp b/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8ae015f43c500413e1239f24be2847cd_md5.webp deleted file mode 100644 index db970e68f..000000000 Binary files a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8ae015f43c500413e1239f24be2847cd_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8d2876047f5078dfd49bb28cb7703643_md5.webp b/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8d2876047f5078dfd49bb28cb7703643_md5.webp deleted file mode 100644 index 95df03236..000000000 Binary files a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_8d2876047f5078dfd49bb28cb7703643_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_eb91b6c0aa14997e1a88191e1acaa8dd_md5.webp b/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_eb91b6c0aa14997e1a88191e1acaa8dd_md5.webp deleted file mode 100644 index 05e002d67..000000000 Binary files a/topics/engineering/assets/infinite-image-gallery-with-r3f---an-approach_eb91b6c0aa14997e1a88191e1acaa8dd_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/introduction-to-software-craftsmanship_9484dea11666b2ee45cd5a11769bde32_md5.webp b/topics/engineering/assets/introduction-to-software-craftsmanship_9484dea11666b2ee45cd5a11769bde32_md5.webp deleted file mode 100644 index 922cb984b..000000000 Binary files a/topics/engineering/assets/introduction-to-software-craftsmanship_9484dea11666b2ee45cd5a11769bde32_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/introduction-to-software-craftsmanship_f4dd9cc532dd7e1845e424a9274811d9_md5.webp b/topics/engineering/assets/introduction-to-software-craftsmanship_f4dd9cc532dd7e1845e424a9274811d9_md5.webp deleted file mode 100644 index b3794672d..000000000 Binary files a/topics/engineering/assets/introduction-to-software-craftsmanship_f4dd9cc532dd7e1845e424a9274811d9_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_2d2ef1b3abadb1e298b1cde0c5614f6a_md5.webp b/topics/engineering/assets/istio_2d2ef1b3abadb1e298b1cde0c5614f6a_md5.webp deleted file mode 100644 index e8a3e89ea..000000000 Binary files a/topics/engineering/assets/istio_2d2ef1b3abadb1e298b1cde0c5614f6a_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_44d2292a4e7d6391c544bfb68ad30f41_md5.webp b/topics/engineering/assets/istio_44d2292a4e7d6391c544bfb68ad30f41_md5.webp deleted file mode 100644 index 21171a1ed..000000000 Binary files a/topics/engineering/assets/istio_44d2292a4e7d6391c544bfb68ad30f41_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_59b3a1cd6fd186a9203774101e1f6ab6_md5.webp b/topics/engineering/assets/istio_59b3a1cd6fd186a9203774101e1f6ab6_md5.webp deleted file mode 100644 index 83aa6bbbd..000000000 Binary files a/topics/engineering/assets/istio_59b3a1cd6fd186a9203774101e1f6ab6_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_5ad7aeee9b729f36ab6d6a0cb8c33acc_md5.webp b/topics/engineering/assets/istio_5ad7aeee9b729f36ab6d6a0cb8c33acc_md5.webp deleted file mode 100644 index 2eaf47016..000000000 Binary files a/topics/engineering/assets/istio_5ad7aeee9b729f36ab6d6a0cb8c33acc_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_72c7525c7279dec224c4d4a2fe92915e_md5.webp b/topics/engineering/assets/istio_72c7525c7279dec224c4d4a2fe92915e_md5.webp deleted file mode 100644 index 90b48945d..000000000 Binary files a/topics/engineering/assets/istio_72c7525c7279dec224c4d4a2fe92915e_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_76c3d8d9b9ee843b5445a06359a73111_md5.webp b/topics/engineering/assets/istio_76c3d8d9b9ee843b5445a06359a73111_md5.webp deleted file mode 100644 index f51d6639e..000000000 Binary files a/topics/engineering/assets/istio_76c3d8d9b9ee843b5445a06359a73111_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_8bc273d9f58042ffc8b0dc8bcba5c4d7_md5.webp b/topics/engineering/assets/istio_8bc273d9f58042ffc8b0dc8bcba5c4d7_md5.webp deleted file mode 100644 index b3fcc8d32..000000000 Binary files a/topics/engineering/assets/istio_8bc273d9f58042ffc8b0dc8bcba5c4d7_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_b26506ec8999334276375c45f5510191_md5.webp b/topics/engineering/assets/istio_b26506ec8999334276375c45f5510191_md5.webp deleted file mode 100644 index 2adc94b12..000000000 Binary files a/topics/engineering/assets/istio_b26506ec8999334276375c45f5510191_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_c3a18948994827d122dccffab6bb925e_md5.webp b/topics/engineering/assets/istio_c3a18948994827d122dccffab6bb925e_md5.webp deleted file mode 100644 index 5daae563b..000000000 Binary files a/topics/engineering/assets/istio_c3a18948994827d122dccffab6bb925e_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_e9b3b350df578cb36174854b3cf061a7_md5.webp b/topics/engineering/assets/istio_e9b3b350df578cb36174854b3cf061a7_md5.webp deleted file mode 100644 index 2dc086efa..000000000 Binary files a/topics/engineering/assets/istio_e9b3b350df578cb36174854b3cf061a7_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_fea0e8efe47df6b5ee4ec298ca2af085_md5.webp b/topics/engineering/assets/istio_fea0e8efe47df6b5ee4ec298ca2af085_md5.webp deleted file mode 100644 index 5b89e7dc0..000000000 Binary files a/topics/engineering/assets/istio_fea0e8efe47df6b5ee4ec298ca2af085_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/istio_ff46797937114ab9f237e8a4e6c75717_md5.webp b/topics/engineering/assets/istio_ff46797937114ab9f237e8a4e6c75717_md5.webp deleted file mode 100644 index 8309b0774..000000000 Binary files a/topics/engineering/assets/istio_ff46797937114ab9f237e8a4e6c75717_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/lessons-learned-from-concurrency-practices-in-blockchain-projects_d24c06b91424a0367b9728cd76f4c3fc_md5.webp b/topics/engineering/assets/lessons-learned-from-concurrency-practices-in-blockchain-projects_d24c06b91424a0367b9728cd76f4c3fc_md5.webp deleted file mode 100644 index 31f2ec8db..000000000 Binary files a/topics/engineering/assets/lessons-learned-from-concurrency-practices-in-blockchain-projects_d24c06b91424a0367b9728cd76f4c3fc_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_format.webp b/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_format.webp deleted file mode 100644 index f0825581d..000000000 Binary files a/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_format.webp and /dev/null differ diff --git a/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_sample.webp b/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_sample.webp deleted file mode 100644 index e4de67a61..000000000 Binary files a/topics/engineering/assets/quality-assurance-works-in-the-product-team_be_fe_testing_doc_sample.webp and /dev/null differ diff --git a/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_doc_sample.webp b/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_doc_sample.webp deleted file mode 100644 index f01204fad..000000000 Binary files a/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_doc_sample.webp and /dev/null differ diff --git a/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_document_format.webp b/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_document_format.webp deleted file mode 100644 index 92b17f031..000000000 Binary files a/topics/engineering/assets/quality-assurance-works-in-the-product-team_mobile_testing_document_format.webp and /dev/null differ diff --git a/topics/engineering/assets/radio-talk-61-monorepo_31a12727d33e9854fbded1b9fbe36668_md5.webp b/topics/engineering/assets/radio-talk-61-monorepo_31a12727d33e9854fbded1b9fbe36668_md5.webp deleted file mode 100644 index 9e2750089..000000000 Binary files a/topics/engineering/assets/radio-talk-61-monorepo_31a12727d33e9854fbded1b9fbe36668_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/radix-sort.gif b/topics/engineering/assets/radix-sort.gif deleted file mode 100644 index 6088421c1..000000000 Binary files a/topics/engineering/assets/radix-sort.gif and /dev/null differ diff --git a/topics/engineering/assets/redis-rate-limiter_gwwpbql.webp b/topics/engineering/assets/redis-rate-limiter_gwwpbql.webp deleted file mode 100644 index b9b083208..000000000 Binary files a/topics/engineering/assets/redis-rate-limiter_gwwpbql.webp and /dev/null differ diff --git a/topics/engineering/assets/redis-rate-limiter_rkmmjtw.webp b/topics/engineering/assets/redis-rate-limiter_rkmmjtw.webp deleted file mode 100644 index 0d66db8f6..000000000 Binary files a/topics/engineering/assets/redis-rate-limiter_rkmmjtw.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_096e90b331b560e88f267337c92af6a4_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_096e90b331b560e88f267337c92af6a4_md5.webp deleted file mode 100644 index 26de9f7fb..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_096e90b331b560e88f267337c92af6a4_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_1a275995943ea3ccb70e132a50132b62_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_1a275995943ea3ccb70e132a50132b62_md5.webp deleted file mode 100644 index 650f07373..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_1a275995943ea3ccb70e132a50132b62_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_8bdc47d5f231e0cb5568bd2ee69ae639_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_8bdc47d5f231e0cb5568bd2ee69ae639_md5.webp deleted file mode 100644 index 00bde3acb..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_8bdc47d5f231e0cb5568bd2ee69ae639_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_9b9fbacb59ea16e62b05e8a8cd7d461a_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_9b9fbacb59ea16e62b05e8a8cd7d461a_md5.webp deleted file mode 100644 index a024c34c8..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_9b9fbacb59ea16e62b05e8a8cd7d461a_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_ba8cd8431044e571ad786c4f3b208125_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_ba8cd8431044e571ad786c4f3b208125_md5.webp deleted file mode 100644 index 0ec176519..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_ba8cd8431044e571ad786c4f3b208125_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/remote-moderated-usability-testing_c758a02d96ab191760c143e5770e7d9b_md5.webp b/topics/engineering/assets/remote-moderated-usability-testing_c758a02d96ab191760c143e5770e7d9b_md5.webp deleted file mode 100644 index 4d5474cb6..000000000 Binary files a/topics/engineering/assets/remote-moderated-usability-testing_c758a02d96ab191760c143e5770e7d9b_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/reusability-in-software-development_54089293152bc5fd40b4e0ddb7dd69e7_md5.webp b/topics/engineering/assets/reusability-in-software-development_54089293152bc5fd40b4e0ddb7dd69e7_md5.webp deleted file mode 100644 index 10f02138f..000000000 Binary files a/topics/engineering/assets/reusability-in-software-development_54089293152bc5fd40b4e0ddb7dd69e7_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/reusability-in-software-development_ea401e3ee43cf4ee90e7edc92fe83900_md5.webp b/topics/engineering/assets/reusability-in-software-development_ea401e3ee43cf4ee90e7edc92fe83900_md5.webp deleted file mode 100644 index e9ca42e74..000000000 Binary files a/topics/engineering/assets/reusability-in-software-development_ea401e3ee43cf4ee90e7edc92fe83900_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_121f206d154f23e1049f0edd39e921cf_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_121f206d154f23e1049f0edd39e921cf_md5.webp deleted file mode 100644 index ebbe431c4..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_121f206d154f23e1049f0edd39e921cf_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_2c5adf742029b611a1459c48c5d1ccb8_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_2c5adf742029b611a1459c48c5d1ccb8_md5.webp deleted file mode 100644 index 9caa8f3d5..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_2c5adf742029b611a1459c48c5d1ccb8_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_3a385622cfd9745a70201ab14b616a09_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_3a385622cfd9745a70201ab14b616a09_md5.webp deleted file mode 100644 index 2b0be58d8..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_3a385622cfd9745a70201ab14b616a09_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_40fe66c43cf65cacbed33cf70df11995_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_40fe66c43cf65cacbed33cf70df11995_md5.webp deleted file mode 100644 index cfd165e79..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_40fe66c43cf65cacbed33cf70df11995_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_8d20afb24ee3dfc8607352f6574e549a_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_8d20afb24ee3dfc8607352f6574e549a_md5.webp deleted file mode 100644 index 2feddd702..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_8d20afb24ee3dfc8607352f6574e549a_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_c5d490b4e34e3a156230e0f702348538_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_c5d490b4e34e3a156230e0f702348538_md5.webp deleted file mode 100644 index 8bc3f9dd5..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_c5d490b4e34e3a156230e0f702348538_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_c9f99108433a5d2449ef51fb884a23dd_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_c9f99108433a5d2449ef51fb884a23dd_md5.webp deleted file mode 100644 index 05e2c6a66..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_c9f99108433a5d2449ef51fb884a23dd_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_fce9cd7be98a3a0133ae89129f323211_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_fce9cd7be98a3a0133ae89129f323211_md5.webp deleted file mode 100644 index c68439f16..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_fce9cd7be98a3a0133ae89129f323211_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-development-life-cycle-101_fdea8ce098bbc365ecd7ffaaec9010a4_md5.webp b/topics/engineering/assets/software-development-life-cycle-101_fdea8ce098bbc365ecd7ffaaec9010a4_md5.webp deleted file mode 100644 index 8fef7a8f2..000000000 Binary files a/topics/engineering/assets/software-development-life-cycle-101_fdea8ce098bbc365ecd7ffaaec9010a4_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/software-quality-assurance_agile.webp b/topics/engineering/assets/software-quality-assurance_agile.webp deleted file mode 100644 index 56d673216..000000000 Binary files a/topics/engineering/assets/software-quality-assurance_agile.webp and /dev/null differ diff --git a/topics/engineering/assets/software-quality-assurance_errors-cost.webp b/topics/engineering/assets/software-quality-assurance_errors-cost.webp deleted file mode 100644 index 5f249f625..000000000 Binary files a/topics/engineering/assets/software-quality-assurance_errors-cost.webp and /dev/null differ diff --git a/topics/engineering/assets/software-quality-assurance_the-stage-of-software-testing.webp b/topics/engineering/assets/software-quality-assurance_the-stage-of-software-testing.webp deleted file mode 100644 index 4d045893a..000000000 Binary files a/topics/engineering/assets/software-quality-assurance_the-stage-of-software-testing.webp and /dev/null differ diff --git a/topics/engineering/assets/split-and-reuse-code-in-react-application_322e43365b34b2a13564d85a3adddcae_md5.webp b/topics/engineering/assets/split-and-reuse-code-in-react-application_322e43365b34b2a13564d85a3adddcae_md5.webp deleted file mode 100644 index e4c64ce56..000000000 Binary files a/topics/engineering/assets/split-and-reuse-code-in-react-application_322e43365b34b2a13564d85a3adddcae_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/split-and-reuse-code-in-react-application_5f86abdbff47c3d17d6258e3b001ceb8_md5.webp b/topics/engineering/assets/split-and-reuse-code-in-react-application_5f86abdbff47c3d17d6258e3b001ceb8_md5.webp deleted file mode 100644 index 8920b506f..000000000 Binary files a/topics/engineering/assets/split-and-reuse-code-in-react-application_5f86abdbff47c3d17d6258e3b001ceb8_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/team-toplogies_3-team-interaction-modes.webp b/topics/engineering/assets/team-toplogies_3-team-interaction-modes.webp deleted file mode 100644 index e1111c97b..000000000 Binary files a/topics/engineering/assets/team-toplogies_3-team-interaction-modes.webp and /dev/null differ diff --git a/topics/engineering/assets/team-toplogies_four-fundamental-topologies-shown-with-the-flow-of-change.webp b/topics/engineering/assets/team-toplogies_four-fundamental-topologies-shown-with-the-flow-of-change.webp deleted file mode 100644 index 1bcd75355..000000000 Binary files a/topics/engineering/assets/team-toplogies_four-fundamental-topologies-shown-with-the-flow-of-change.webp and /dev/null differ diff --git a/topics/engineering/assets/team-toplogies_the-four-fundamental-team-topologies.webp b/topics/engineering/assets/team-toplogies_the-four-fundamental-team-topologies.webp deleted file mode 100644 index 8475a4abf..000000000 Binary files a/topics/engineering/assets/team-toplogies_the-four-fundamental-team-topologies.webp and /dev/null differ diff --git a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka-controller-apis.webp b/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka-controller-apis.webp deleted file mode 100644 index 8f985ee27..000000000 Binary files a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka-controller-apis.webp and /dev/null differ diff --git a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka_architecture.webp b/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka_architecture.webp deleted file mode 100644 index b69fadb2c..000000000 Binary files a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka_architecture.webp and /dev/null differ diff --git a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_time-shutdown-operations-kafka.webp b/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_time-shutdown-operations-kafka.webp deleted file mode 100644 index c7a10d3ea..000000000 Binary files a/topics/engineering/assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_time-shutdown-operations-kafka.webp and /dev/null differ diff --git a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-javascript.gif b/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-javascript.gif deleted file mode 100644 index 50beae6fc..000000000 Binary files a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-javascript.gif and /dev/null differ diff --git a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-ocaml.gif b/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-ocaml.gif deleted file mode 100644 index 8265e2434..000000000 Binary files a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-ocaml.gif and /dev/null differ diff --git a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-python.gif b/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-python.gif deleted file mode 100644 index 3591e3622..000000000 Binary files a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-python.gif and /dev/null differ diff --git a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-shell.gif b/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-shell.gif deleted file mode 100644 index 8b2c38915..000000000 Binary files a/topics/engineering/assets/vim-repl-driven-development_repl-driven-development-shell.gif and /dev/null differ diff --git a/topics/engineering/assets/well-crafted-software_871104804c16cbc9ea337a0b8c851035_md5.webp b/topics/engineering/assets/well-crafted-software_871104804c16cbc9ea337a0b8c851035_md5.webp deleted file mode 100644 index 99bcf6e8d..000000000 Binary files a/topics/engineering/assets/well-crafted-software_871104804c16cbc9ea337a0b8c851035_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/what-is-kubernetes_ca7490cb6fe10475c91a359a8177d29c_md5.webp b/topics/engineering/assets/what-is-kubernetes_ca7490cb6fe10475c91a359a8177d29c_md5.webp deleted file mode 100644 index 40525ebd8..000000000 Binary files a/topics/engineering/assets/what-is-kubernetes_ca7490cb6fe10475c91a359a8177d29c_md5.webp and /dev/null differ diff --git a/topics/engineering/assets/working-on-a-project-interview-assessment-at-dwarves_3544e2b2c437826a3005b95909ec2795_md5.gif b/topics/engineering/assets/working-on-a-project-interview-assessment-at-dwarves_3544e2b2c437826a3005b95909ec2795_md5.gif deleted file mode 100644 index 529a6b9a0..000000000 Binary files a/topics/engineering/assets/working-on-a-project-interview-assessment-at-dwarves_3544e2b2c437826a3005b95909ec2795_md5.gif and /dev/null differ diff --git a/topics/engineering/burnup-and-burndown-chart.md b/topics/engineering/burnup-and-burndown-chart.md deleted file mode 100644 index 2a14e9f80..000000000 --- a/topics/engineering/burnup-and-burndown-chart.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Burnup and burndown chart -description: Learn how Burnup and Burndown charts help project managers track product development progress by showing completed work, remaining tasks, and timeline status clearly to clients. -date: 2023-02-28 -github_id: nhuthm -icy: 10 -redirect: - - /RmRrkw ---- - -From the team product, when the Project Manager want to report with their client for the question: "How is our project coming along?", to have a catch-up on the product's process. - -Then, definitely, they will need some type of the report like the specific chart, to show with their client on the progress for the product development, instead of just saying something like: "Everything is on track". - -There are 2 different types of chart can follow up: Burnup and Burndown. - -## What is a Burndown chart? - -For this chart, we are using for visualizing the amount of work left to complete in a specific project development, to see if how quickly a team is moving forward to reach the common goal. - -![](assets/burnup-and-burndown-chart_burndown_chart_example.webp) - -Based on the reference documents, those are 4 items to show how and when we use the Burndown chart: - -> - Actual velocity or speed of the entire team -> - Estimated speed per sprint -> - Total work complete at each point in time -> - Remaining tasks versus time remaining - -## What is a Burnup chart? - -With this chart, it is used to track how much of a specific project, features has been completed within a planning timeline. To see if the product team is making thing on the exact timeline that they had planned. - -![](assets/burnup-and-burndown-chart_burnup_chart.webp) - -Same with the Burndown chart, we will have 4 items to show how and when we use the Burnup chart as well: - -> - The number of overall tasks completed -> - The amount of time each task took to complete -> - If a project is on time -> - Work that is added to the scope or into an existing sprint - -## Differences between burndown and burnup - -Basically, when wanting to see what is left to be completed on the giving timeline, we can use Burndown chart to track. On the other hands, if we want to see what are the things that the team had done to highlight, also for checking if there any popup work is added into the timeline to have an impact on it. - -## Reference - -- [Burndown and burnup charts: what's the difference and how to use them](https://rindle.com/blog/burndown-and-burnup-charts-whats-the-difference-and-how-to-use-them) -- [Burndown vs burnup chart](https://www.projectmanagement.com/blog/blogPostingView.cfm?blogPostingID=40731&thisPageURL=/blog-post/40731/Burndown-vs-Burnup-Chart#_=_) diff --git a/topics/engineering/cognitive-load.md b/topics/engineering/cognitive-load.md deleted file mode 100644 index 54144c09b..000000000 --- a/topics/engineering/cognitive-load.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Cognitive load -description: Learn how cognitive load affects mental effort and team performance, and why managing it is key to reducing overload and improving focus in business and technology teams. -date: 2022-08-28 -github_id: zlatanpham -redirect: - - /Dw0XWQ ---- - -Cognitive load is a measure of how much mental effort is required for a task. It’s usually thought of as the amount of information you have to process at any given moment. For example, if you’re trying to solve a math problem, you have to hold in mind that 2+2=4, you need to know what 2 and 4 are, and you need to know how to add numbers together. That’s three separate things that your brain needs to remember at the same time. - -Cognitive load can also be measured by asking people about how much effort they think they’ll need for a given task. If someone thinks that doing something will take a lot of cognitive effort, then it probably will. - -When we talk about cognitive load, it’s easy to understand that any one person has a limit on how much information they can hold in their brains at any given moment. The same happens for any one team by simply adding up all the team members’ cognitive capacities. However, we rarely take into account cognitive load when assigning responsibilities or software parts to a given team. Perhaps because it’s difficult to quantify both the available capacity and what the cognitive load will be. Or perhaps this is because teams are expected to adapt to changes in their tasks, with no questions asked. When cognitive load is not considered, teams spread themselves thin by trying to cover a lot of responsibilities and areas. Such teams lack the bandwidth to pursue mastery of their trade and struggle with high context switching costs. - -## Reference - -- _Team Topologies: Organizing Business and Technology Teams for Fast Flow by Manuel Pais and Matthew Skelton_ diff --git a/topics/engineering/cold-start-problem.md b/topics/engineering/cold-start-problem.md deleted file mode 100644 index be3599f6e..000000000 --- a/topics/engineering/cold-start-problem.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Cold start problem -description: Learn how to reduce serverless function cold start delays by keeping containers warm with scheduled requests, improving performance for faster response times and better user experience. -date: 2022-03-28 -github_id: zlatanpham -redirect: - - /roHRZw ---- - -One of the biggest concern against Serverless Function is the problem of cold start. The first cold start happens when the first request comes in after deployment. After that request is handled, the instance stays alive to be reused by the upcoming requests. If the function, then, has not been invoked in a certain amount of time, it will change back to the cold state. In particular, the invocation steps are: - -1. Get the code from persistent storage -2. Spin up the container -3. Load the package in memory -4. Run the function - -The cold start time is around 0.5s to 2s but increases based on the function size (step 1 to 3). When the container is already warm, it jumps right to step 4. - -**How to resolve it** - -While keeping the function size small or increasing the memory could partly speed up the load time, the latency is still in complain because > 1s request cannot ensure a good user experience. One way to fix it is to make sure the container is always in "hot" state. This can be achieved by running a scheduler to send a request to reset the cycle time of the function. - -#### References - -- -- diff --git a/topics/engineering/conway-s-law.md b/topics/engineering/conway-s-law.md deleted file mode 100644 index 93abb0dd0..000000000 --- a/topics/engineering/conway-s-law.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Conway s law -description: "Conway's Law explains how a company's communication structure shapes its software design, highlighting the need to align team organization with system architecture for better development outcomes." -date: 2022-06-12 -github_id: zlatanpham -redirect: - - /zLgLEw ---- - -> Any organization that designs a system will produce a **design** whose structure is a **copy of the organization's communication structure**. -> -> -- Conway's Law - -In reverse words, the way a company or a team is structured will determine how its software structures and works. The law is so important because it reveals a lot of important facets that we might want to examine before building software: - -- It tells us that we should always start with people first. -- It tells us that we can't change the architecture of a software unless we change the same time how the people working on it are organized. -- It tells us to look at the people to understand why a certain system was developed in a certain way. -- It tells us the two modules developed by two development groups cannot interface with each other unless there are communications between those groups. -- It tells us to not panic if experiencing a large group starts breaking into smaller ones because it's how an organization grows by reducing communication overhead. -- It tells us that organizational hierarchies should change as fast as either the need for internal innovation or external market pressure. - -#### References - -- https://intenseminimalism.com/2013/conways-law/ -- http://www.melconway.com/Home/Conways_Law.htmlConway diff --git a/topics/engineering/daemons-and-services-programming-guide.md b/topics/engineering/daemons-and-services-programming-guide.md deleted file mode 100644 index 88c92be83..000000000 --- a/topics/engineering/daemons-and-services-programming-guide.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Daemons and services programming guide -description: Learn how to design and manage macOS daemons and services, including login items, XPC services, launch agents, and daemons, with best practices for background processes and interprocess communication. -date: 2020-05-11 -tags: - - macos -redirect: - - /Pai7GA ---- - -## About daemons and services - -Many kinds of tasks that do not require user interaction are most effectively handled by a process that runs in the background. You can use a daemon or service to: - -**Provide server functionality**, such serving web pages. Coordinate access to of a shared resource, such as a database. Perform work for a foreground application, such as file system access. - -![](assets/daemons-and-services-programming-guide_8f72ec64cac909319e7fc79210c69563_md5.webp) - -**Terminology Note**: The terms `service` and `daemon` have several meanings in different contexts, with further variation over time and from one development community to another. - -In this document, service refers to a background process that supports a full GUI application in some way, for example by registering a global hotkey or by performing network communication. Daemon refers to all other types of background processes, especially those that don’t present any kind of user interface. - -## Designing daemons and services - -Two of the most important design decisions to consider when creating a background process are how it will be run and how other processes will communicate with it. These two considerations interact with each other: different types of background processes have different forms of communication available to them. - -## Types of background processes - -There are four types of background processes in OS X. The differences are summarized in Table 1-1 and described in detail in the following subsections. To select the appropriate type of background process, consider the following: - -Whether it does something for the currently logged in user or for all users. Whether it will be used by single application or by multiple applications. Whether it ever needs to display a user interface or launch a GUI application. - -![](assets/daemons-and-services-programming-guide_93dcafb893273d168521685e97d42ad8_md5.webp) - -/Login items are started by the per-user instance of launchd, but it doesn't take any actions to manage them. - -### Login items - -Login items are launched when the user logs in, and continue running until the user logs out or manually quits them. Their primary purpose is to allow users to open frequently-used applications automatically, but they can also be used by application developers. For example, a login item can be used to display a menu extra or to register a global hotkey. - -For example, many to-do applications use a login item that listens for a global hotkey and presents a minimal UI allowing the user to enter a new task. Login items are also commonly used to display user interface items, such as a floating clock or a timer, or to display an icon in the menu bar. . - -Another example is a calendaring application with a helper application launched as a login item. The helper application runs in the background and launches the main GUI application when appropriate to remind the user of upcoming appointments - -### XPC services - -XPC services are managed by launchd and provide services to a single application. They are typically used to divide an application into smaller parts. This can be used to improve reliability by limiting the impact if a process crashes, and to improve security by limiting the impact if a process is compromised. - -With traditional single-executable applications, if any part of the application crashes, the entire application terminates. By restructuring the application into a main process and services, the impact of a crash in a service is significantly less. The user can continue to work; the service that crashed gets relaunched. For example, an email application can use an XPC service to handle communication with the mail server. Even if the service crashes, temporarily interrupting communication with the server, the rest of the application remains usable. - -Sandboxing allows you to specify a list of things your program is expected to do during normal operation. The operating system enforces that list, limiting the damage that can by done by an attacker. For example, a text editor might need to edit files on disk that have been opened by the user, but it probably does not need to open arbitrary files in other locations or communicate over the network. - -You can combine sandboxing with XPC services to provide privilege separation by splitting a complex application, tool, or daemon into smaller pieces with well-defined functionality. Because of the reduced privileges of of each individual piece, any flaws are less exploitable by an attacker: none of the pieces run with the full capabilities of the user. For example, an application that organizes and edits photographs does not usually need network access. However, if it also allows users to upload photos to a photo sharing website, that functionality can be implemented as an XPC service with network access and mediated access (or no access) to the file-system. - -### Launch daemons - -Daemons are managed by launchd on behalf of the OS in the system context, which means they are unaware of the users logged on to the system. A daemon cannot initiate contact with a user process directly; it can only respond to requests made by user processes. Because they have no knowledge of users, daemons also have no access to the window server, and thus no ability to post a visual interface or launch a GUI application. Daemons are strictly background processes that respond to low-level requests. - -Most daemons run in the system context of the system—that is, they run at the lowest level of the system and make their services available to all user sessions. Daemons at this level continue running even when no users are logged into the system, so the daemon program should have no direct knowledge of users. Instead, the daemon must wait for a user program to contact it and make a request. As part of that request, the user program usually tells the daemon how to return any results - -### Launch agent - -Agents are managed by launchd, but are run on behalf of the currently logged-in user (that is, in the user context). Agents can communicate with other processes in the same user session and with system-wide daemons in the system context. They can display a visual interface, but this is not recommended. - -If your code provides both user-specific and user-independent services, you might want to create both a daemon and an agent. Your daemon would run in the system context and provide the user-independent services while an instance of your agent would run in each user session. The agents would coordinate with the daemon to provide the services to each user. - -## Protocols for communicating with daemons - -There are four major communication mechanisms commonly used between daemons and their clients: XPC, traditional client-server communications (including Apple events, TCP/IP, UDP, other socket and pipe mechanisms), remote procedure calls (including Mach RPC, Sun RPC, and Distributed Objects), and memory mapping (used underneath the Core Graphics APIs, among others). - -XPC is the easiest way to launch and communicate with your daemon. For details on implementing this mechanism, see Creating XPC services and XPC services API Reference. - -Other RPC (remote procedure call) mechanisms such as Distributed Objects should be avoided for communication across security domain boundaries, for example a user process communicating with a system-level daemon, because this creates a security risk. They are appropriate only when you can be certain that both processes involved have the same level of privileges. - -In most other cases, you should use a traditional client-server communication API. Code based on these APIs tends to be easier to understand, debug, and maintain than RPC or memory mapping designs. It also tends to be more portable to other platforms than RPC-based code. - -Memory mapping requires complex management and represents a security risk if you are not careful about what memory pages you share or if you do not sufficiently validate the shared data. You should use memory mapping only if your client and daemon require a large amount of shared state with low latency, such as passing audio or video in real time. - -## Viewing the currently running daemons - -If you want to see the daemons currently running on your system, use the Activity Monitor application (located in /Applications/Utilities). This application lets you view information about all processes including their resource usage. Figure 1-1 shows the Activity Monitor window and the process information. - -Note: If you want to know more about the services provided by a particular daemon, consult the man page for that daemon. You can also view the manual pages online by reading OS X Man Pages. - -![](assets/daemons-and-services-programming-guide_6b7ebc46edfa8c99dbdbfb540f15bb11_md5.webp) diff --git a/topics/engineering/difference-between-project-program-portfolio-manager.md b/topics/engineering/difference-between-project-program-portfolio-manager.md deleted file mode 100644 index b7d861be4..000000000 --- a/topics/engineering/difference-between-project-program-portfolio-manager.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Difference between project program portfolio manager -description: Learn the key differences between Project Manager, Program manager, and Portfolio manager roles in software development and how they manage teams to meet client requirements effectively. -date: 2023-01-17 -github_id: nhuthm -icy: 10 -redirect: - - /XIr61w ---- - -When working in the Software Development team, especially for those Product team, we might recognize several manager roles: Project manager - product manager, Program manager, and Portfolio manager. On the Internet, we can find those definition for them, on this note, it just a short summarize on what I understand after reading from the specific source. - -## Project manager - product manager - -For this person, we can see that he/she will be the one who is incharge of planning, scheduling, budgeting, execution, and delivery of the product. Basically, they are the one who will run the team based on their leadership skills, technical knowledge, and experiences to direct their members. Also, the most important is, the software is meet with the client's requirements. - -## Program manager - -In the Product team, we might have quite a lot of team to work on the different features that are working on the big system. Right now, the Program manager is the one who is incharge of the work for the team project. Basically, for this role, it's just higher than Project Manager. If the Project Manager is the one who want to make sure the project's goal reach client's requirement, the Program manager is the one who will decide which team will work on the requirement. - -For example: if your product are having about 5 or 6 teams, the Program manager is assigned to manage the team number 2nd and 3rd. When receiving the request, the Program manager will think about which team that is he/she is incharged, to work on this request so the team can deliver successfully. - -## Portfolio manager - -Can say, this role is the most important than others 2 mention roles above. The reason is: they will be the one who have to prioritize the work for every teams, every programs that they are managing. After that, they will know which program should do the stuffs before providing informations to project team. Basically, can see that they are mostly focusing on the business site to balance the workloads. - -## The difference between those roles - -Basically, for those 3 roles: Project Manager, Program manager, and Portfolio manager are in common of 1 term: Management. The big difference between them is: the responsible for each role that is playing in the Product team to make everything is working correctly and the output is reach with the client's requirements. - -## References - -- https://www.northeastern.edu/graduate/blog/project-management-vs-portfolio-management-vs-program-management/ diff --git a/topics/engineering/double-entry-accounting.md b/topics/engineering/double-entry-accounting.md deleted file mode 100644 index 0cf638e39..000000000 --- a/topics/engineering/double-entry-accounting.md +++ /dev/null @@ -1,244 +0,0 @@ ---- -title: Double entry accounting -description: Learn the basics of double-entry accounting, including assets, liabilities, equity, and how this system improves accuracy over single-entry bookkeeping for better financial tracking and reporting. -date: 2022-08-24 -github_id: monotykamary -redirect: - - /F6NH0w ---- - -## Introduction - -Double-entry accounting states that every financial transaction has equal and opposite effects in at least two different accounts. There are 3 main accounting types associated with this: _assets_, _liabilities_, and _equity_. - -Accountants have an **accounting equation**, and it is used to check whether the bookkeeper has made a mistake if it is ever out of balance. This equation is commonly used in balance sheets/statement of affairs: - -$$ -\text{Assets} = \text{Liabilities} + \text{Equity} -$$ - -> [!INFO] We can derive a person's net worth to be the total equity they have: -> -> $$ -> \text{Equity (Net Worth)} = \text{Assets} - \text{Liabilities} -> $$ - -### How is it different from single-entry? - -Single-entry accounting involves writing down all entity transactions (revenues, expenses, payroll, etc.) in a single ledger. The database equivalent is a list of transactions that add or subtract money from an account. This lack of detail in recording makes it **difficult** to track assets and liabilities, and it is prone to mistakes. - -### Single-entry example - -The single-entry example allows us to calculate a closing balance, but it doesn't have the granularity to tell us whether this balance is from our bank balance or our total commodities. - -| Date | Details | Amount (USD) | -| ------------ | ---------------------------- | ------------ | -| Jan 1, 2020 | Opening Balance | 0.00 | -| Jan 5, 2020 | Initial Investment | 5000 | -| Jan 9, 2020 | MacBook purchase | -2000 | -| Jan 11, 2020 | Domain Registration (1 year) | -35 | -| Jan 14, 2020 | Web Hosting Fees (1 year) | -100 | -| Jan 16, 2020 | Advertising Expense | -20 | -| Jan 18, 2020 | Product Sales | 50 | -| Jan 20, 2020 | Bank Fees | -2 | -| Jan 23, 2022 | Product Sales | 100 | -| Jan 27, 2022 | Taxes | -300 | -| | | | -| | **Closing Balance** | 2693 | - - - -### Double-entry example - -The double-entry example allows us to derive balances for our assets and liabilities on top of checking whether our assets, liabilities, and equity balance each other out. - -| Date | Details | Label (Account) | Debit (USD) | Credit (USD) | -| ------------ | ---------------------------- | --------------------------- | ----------- | ------------ | -| Jan 1, 2020 | Opening Balance | Assets:Bank | 0 | | -| | | Equity:Opening Balance | | 0 | -| Jan 5, 2020 | Initial Investment | Assets:Bank | 5000 | | -| | | Equity:Opening Balance | | 5000 | -| Jan 9, 2020 | MacBook purchase | Equity:Expenses:Electronics | 2000 | | -| | | Assets:Bank | | 2000 | -| Jan 11, 2020 | Domain Registration (1 year) | Equity:Expenses:Web | 35 | | -| | | Liabilities:Credit Card | | 35 | -| Jan 14, 2020 | Web Hosting Fees (1 year) | Equity:Expenses:Web | 100 | | -| | | Liabilities:Credit Card | | 100 | -| Jan 16, 2020 | Advertising Expense | Equity:Expenses:Marketing | 20 | | -| | | Assets:Bank | | 20 | -| Jan 18, 2020 | Product Sales | Assets:Bank | 50 | | -| | | Equity:Revenue | | 50 | -| Jan 20, 2020 | Bank Fees | Equity:Expenses | 2 | | -| | | Assets:Bank | | 2 | -| Jan 23, 2022 | Product Sales | Assets:Bank | 100 | | -| | | Equity:Revenue | | 100 | -| Jan 27, 2022 | Sales Tax | Liabilities:Taxes:Sales | 15 | | -| | | Assets:Bank | | 15 | -| | | | | | -| | | **Total** | 7322 | 7322 | - - - - -### Reasons and proponents for accounting in double-entry - -In a double-entry accounting system, a debit in one account offsets a credit in another, the sum of **all debits** must equal the sum of **all credits**. The system standardizes the accounting process and improves the accuracy of prepared financial statements, and enables us to detect errors in cases of fraud or laundering. - -Double-entry systems, regardless of whether we keep track of journal entries, allow us to create reports on income statements, balance sheets, statements of cash flows, and statements of retained earnings. - -## Database design - -For database design, refer to [[Database design for double-entry accounting | database design for double-entry accounting]]. - ---- - -## Accounting types - -### Assets - -> **"How much do I have?"** - -Assets refer to anything of value that an entity owns. They are represented as a **debit balance**. Assets are generally divided into 2 categories: - -- **Current assets**: anything that can be consumed, sold, or converted into cash within a year - - _Inventory_: stocked goods you intend to sell - - _Receivables_: payments your clients and customers owe you - - _Cheques_: a document that orders a bank to pay out money to a person's account -- **Fixed assets**: assets which are purchased for long-term use and are not likely to be converted quickly into cash within a year; these include, but are not limited to: buildings, land, machinery, vehicles, software, etc. - -### Liabilities - -> **"How much do I owe?"** - -Liabilities refer to any debts the entity has. They are represented as a **credit balance**. Liabilities are divided into 3 categories: - -- **Current liabilities**: any debts that you owe within the next 12 months - - _Taxes_: a debt owed to a taxing authority; these include, but are not limited to income tax, sales tax, and capital gains tax - - _Credit cards_: a debt owed to the bank through the medium of a scannable card - - _Salaries and wages payable_: agreements of payment to employees as a form of debt - - _Short-term loans_: loans taken from an institution to be paid within the year - - _Overdrafts_: a deficit in a bank account allowed by a bank to draw more money than the account holds -- **Non-current liabilities**: long-term debt that goes beyond 12 months - - _Long-term loans_: loans with repayment terms usually longer than five years - - _Mortgages_: a loan to purchase or maintain real estate from a financial institution -- **Contingent liabilities**: liabilities that may occur depending on the outcome of a future event - - _Lawsuits_: a claim or dispute brought to the court of law - - _Product warrenties_: a guarantee a manufacture or similar party regarding the condition of the product - -### Equity - -> **"How much is left over?"** - -If we follow the equation for net worth, assets minus liabilities would give us our total equity. However, equity from the perspective of a transaction refers to the value of something. Say, for instance, when you start a ledger, there was $100 in your checking account. Where will that money come from? The answer is **your equity**. - -If you have a kidney worth $262,000, then you have $262,000 in equity in that kidney. To convert the kidney (a commodity) into cash or to **credit** your bank account, you will have to **debit** your kidney by selling it. - -> [!INFO] -> -> Types such as _expenses_, _revenue_, and _income_ are considered **subcategories** of _equity_. - -There are quite a lot of subcategories of equity. This includes, but is not limited to: - -- _Expenses_: the cost of money spent on something -- _Revenue_: money received generated from business operations -- _Income_: money received from an agreement, either through work or investment - -Equity also reflects all kinds of assets that have not been debited as assets. In this case, equity is represented as a **credit balance**. - ---- - -## Terminology - -| Basis for comparison | Debit | Credit | -| -------------------- | ----------------------------------------------------- | ------------------------------------------------------- | -| Meaning | Refers to a record of money flowing _into_ an account | Refers to a record of money flowing _out of_ an account | - -In accounting, every financial transaction of an entity is kept inside a **_journal_**. The entries inside the journal are used to create a general **_ledger_**. The differences between them are listed in the table below: - -| Basis for comparison | Journal | Ledger | -| ---------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------- | -| Meaning | The book which records all financial transactions of an entity | The book that contains financial information needed to make | -| Known as | Book of original entry | Book of secondary entry | -| Purpose | Used in preparation of a ledger | Used for labelling balances for final accounts | -| Transactions Recorded | All entries are made in chronological order | Entries are organized by account | -| Debit and credit | Can be separated by columns or by entries | The left side of a ledger is often the debit side while the right side is credit | -| Narration (comment or description) | Required | Not required | -| Balancing | Balancing is not done | All accounts are balanced based on the 5 main accounting types | -| Granularity | Offers the highest granularity as all transactions are recorded | Offers little granularity in comparison as it only shows account data | - -## Appendix - -### `hledger` for personal use - -[`hledger`](https://hledger.org/) is a multi-currency double-entry accounting software that is accessible through the command-line/terminal. It is mostly inspired by [ledger-cli](https://www.ledger-cli.org/), but rewritten in Haskell with build support for M1 Mac machines. - -```sh -$ hledger -f transactions.journal balance -t - 3113 USD Assets:Bank - -2993 USD Equity - 2157 USD Expenses - 2000 USD Electronics - 20 USD Marketing - 135 USD Web - -5000 USD Opening Balance - -150 USD Revenue - -120 USD Liabilities - -135 USD Credit Card - 15 USD Taxes:Sales --------------------- - 0 -``` - -``` -## journal -2020-01-01 Opening Balance - Assets:Bank 0 USD - Equity:Opening Balance 0 USD - -2020-01-05 Initial Investment - Assets:Bank 5000 USD - Equity:Opening Balance -5000 USD - -2020-01-09 MacBook purchase - Equity:Expenses:Electronics 2000 USD - Assets:Bank -2000 USD - -2020-01-11 Domain Registration (1 year) - Equity:Expenses:Web 35 USD - Liabilities:Credit Card -35 USD - -2020-01-14 Web Hosting Fees (1 year) - Equity:Expenses:Web 100 USD - Liabilities:Credit Card -100 USD - -2020-01-16 Advertising Expense - Equity:Expenses:Marketing 20 USD - Assets:Bank -20 USD - -2020-01-18 Product Sales - Assets:Bank 50 USD - Equity:Revenue -50 USD - -2020-01-20 Bank Fees - Equity:Expenses 2 USD - Assets:Bank -2 USD - -2020-01-23 Product Sales - Assets:Bank 100 USD - Equity:Revenue -100 USD - -2020-01-23 Sales tax - Liabilities:Taxes:Sales 15 USD - Assets:Bank -15 USD -``` - -## Reference - -- -- -- -- -- -- -- . -- diff --git a/topics/engineering/enabling-team.md b/topics/engineering/enabling-team.md deleted file mode 100644 index 979502a29..000000000 --- a/topics/engineering/enabling-team.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Enabling team -description: Learn how enabling teams support stream-aligned teams by bridging skill gaps, sharing expertise, and improving delivery through collaboration, technical guidance, and knowledge transfer. -date: 2022-08-15 -github_id: zlatanpham -redirect: - - /fMa08g ---- - -To succeed in a competitive environment, [[Stream-aligned team | stream-aligned teams]] must continually learn and improve their capabilities. But with constant pressure to deliver and respond to change quickly, how can a stream-aligned team with end-to-end ownership find time to do research, read about new skills, practice them, and then integrate the new knowledge into its service delivery? - -An enabling team is composed of experts who bridge the capability gap between the stream-aligned teams and the technical (or product) domain. Such a team can provide valuable input and advice to the stream-aligned teams to help them acquire and evolve capabilities without having to invest as much effort or time in doing so. - -Enabling teams are strongly collaborative; they thrive to understand the problems and shortcomings of stream-aligned teams in order to provide effective guidance. We would also like to call them “Technical Consulting Teams,” as these teams provide (guidance, not execution), whether internal or external to the organization. - -An enabling team usually plays a key role in providing the stream-aligned team with the tools it needs to be successful. The enabling team might map to any of the stream-aligned team capabilities (user experience, architecture, testing), but often it focuses on build engineering, continuous delivery, deployments or test automation for particular client technology (e.g., desktop, mobile, web). For example, the enabling team might set up a skeleton deployment pipeline or a basic test framework by combining automation tools and some initial scenarios and samples. - -Knowledge transfer can take place on a temporary basis (when a stream-aligned team adopts a new technology, such as containerization) or on a long-term basis (for continuously improving aspects, such as faster builds or faster test execution). Pairing can be quite effective for some types of practices, such as defining Infrastructure-as-Code. - -## Expected behaviors - -The following are types of behaviors and outcomes to expect from an effective enabling team: - -- An enabling team works together with stream-aligned teams to understand their needs, establishing regular checkpoints and agreeing when more collaboration is needed. -- Teams that enable other teams stay abreast of new approaches, tools, and practices in their area of expertise before an actual need is expected from stream-aligned teams. -- An enabling team serves as the messenger to announce both good news—such as a new UI automation framework that can reduce our custom test code by 50%—and bad news—such as the fact that Javascript framework X, which we’re using extensively, is no longer actively maintained. This helps management with the technology life cycle. -- The enabling team might occasionally act as a proxy for other services that stream-aligned teams cannot yet use directly. -- An enabling team promotes learning not only within the enabling team but across organizational stream-aligned teams, acting as a curator that facilitates appropriate knowledge sharing within the organization. It should consider itself as "the key learning function" of the organization. - -The purpose of enabling teams is to help stream-aligned teams deliver working software in a sustainable, responsible way. Enabling teams do not exist to fix problems that arise from poor practices, poor prioritization choices, or poor code quality within stream-aligned teams. stream-aligned teams should expect to work with enabling teams for short periods of time (weeks or months) in order to increase their capabilities around a new technology, concept, or approach. After the new skills and understanding have been embedded in the stream-aligned team, the enabling team will switch its focus to a different team. - -## Reference - -- _Team Topologies: Organizing Business and Technology Teams for Fast Flow by Manuel Pais and Matthew Skelton_ diff --git a/topics/engineering/istio.md b/topics/engineering/istio.md deleted file mode 100644 index 40d78925c..000000000 --- a/topics/engineering/istio.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: Istio -description: Learn how Istio service mesh improves microservices networking with features like traffic management, security, and monitoring, and why it complements API gateways for scalable applications. -date: 2019-06-09 -tags: - - tool -redirect: - - /PtbUfg ---- - -Istio is an implementation of the Service mesh architecture, which is a network of microservices that interactive with each other to form an application. Besides Istio, there are several concept applying service mesh architecture such as Linkerd, Consul. - -## Service mesh vs API gateway - -Service mesh has been really a hot term recently, comparing to another microservice architecture, **API gateway**, which is considered simpler and a more mature solution. Therefore before digging into Istio, it is necessary to have a comparison between Service mesh and API gateway, to find out the pros/cons and the use case of each architecture, thus have a better overview for Istio. - -### API gateway - -The key objective API Gateway is to expose microservices as managed API. API Gateways comes with a number of powerful features such as load balancing, health checks, API versioning and routing, authentication & authorization, data transformation, analytics, logging, SSL termination, etc. Kong, Ambassador is two of successful open source API gateways. - -![](assets/istio_72c7525c7279dec224c4d4a2fe92915e_md5.webp) - -**Pros** - -- Microservices can focus on business logic -- Authentication, logging, and monitoring can be handled by the API Gateway -- Flexibility to use completely independent protocols in which clients and microservice can talk -- Can handle failures/retries - -**Cons** - -- API Gateways are fairly centralized, so it could be a single point of failure if it is not well managed and scaled. -- Although they can be scalable, they still require a single point to register new APIs or change configuration -- From an organizational perspective, they are likely to be maintained by a single team - -### Service mesh - -Unlike API Gateways, Service meshes are focus on decentralized and self-organizing networks between microservices. that handle load balancing, service discovery, health checks, monitoring, and tracing. The mesh work by attaching small agents container, also known as "sidecar" alongside with every instance that manipulate the inbound/outbound traffic and handles instance registration, metric collection, and tracing. Istio, Linkerd are the most known service meshes. - -![](assets/istio_c3a18948994827d122dccffab6bb925e_md5.webp) - -**Pros** - -- Because of the decentralization, service mesh entities handle their own traffic without being gathered at a single point. -- Service meshes are more dynamic and can easily shift shape and accommodate new functionalities and endpoints. -- Their decentralized nature makes it easier to work on microservices for development teams -- Resilience, failure/retry handling is as power as API gateway - -**Cons** - -- Service meshes is complex and require quite lot of resource. -- It requires the deployment of a separate traffic manager, a telemetry gatherer, a certificate manager and a sidecar process for each instance. -- They are still young, and need a lot more of development to be fully ready for a production grade microservice network. - -## A nutshell - -It’s easy to see that both API Gateway and Service mesh have the strength that each other misses, so many developers agree that the best practice for microservices network are combining both of them. Istio is the very first pioneer in this approach, making it the worthiest architecture in the world, that’s why it is backed by many engineers from tech giants like Google, IBM and Redhat. - -## What leads to Istio - -Let's begin with Kubernetes - the famous container orchestration platforms To make a microservices network, k8s basically run these 3 entities: - -- Pod - a group of one or more containers, with shared storage/network -- Deployment - manages pod definition and defines replicas of pods -- Service - an abstraction, an access point to a set of Pods - -So we have the microservices the Kubernetes way: - -![](assets/istio_fea0e8efe47df6b5ee4ec298ca2af085_md5.webp) - -What if I found microservices grow up like this? - -![](assets/istio_ff46797937114ab9f237e8a4e6c75717_md5.webp) - -Definitely it will become a multiple points of failure. This is where the savior Service mesh come in, and Istio can solve the problem. Istio injects a sidecar in every pod in the network: - -![](assets/istio_e9b3b350df578cb36174854b3cf061a7_md5.webp) - -So instead of continuously writing code for the routing, the circuit breaker or every networking stuff, we can focus on the business logic of each application in the network. - -![](assets/istio_8bc273d9f58042ffc8b0dc8bcba5c4d7_md5.webp) - -Our system's network now becomes more under controlled: - -![](assets/istio_5ad7aeee9b729f36ab6d6a0cb8c33acc_md5.webp) - -## Istio architecture - -### Envoy - -Istio utilizes an expanded Envoy proxy version which is a high-performance proxy built in C++ for all facilities in the service mesh to mediate all inbound and outbound traffic of the entire mesh network. Istio leverages many integrated characteristics of Envoy, making it the core component in establishing service meshes, such as: - -- Dynamic service discovery -- Load balancing -- TLS termination -- HTTP/2 and gRPC proxies -- Circuit breakers -- Health checks -- Staged rollouts with percentage based traffic split -- Fault injection -- Rich metrics - -![](assets/istio_2d2ef1b3abadb1e298b1cde0c5614f6a_md5.webp) - -Envoy is deployed along side with every Kubernetes pods as a sidecar to the appropriate proxy. This deployment allows Istio to extract information about traffic behavior as attributes. Istio can, in turn, use these attributes in Mixer to enforce policy decisions, and send them to monitoring systems to provide information about the behavior of the entire mesh. - -### Pilot - -Pilot provides service discovery for the Envoy sidecars, traffic management capabilities for intelligent routing (e.g., A/B tests, canary rollouts, etc.), and resiliency (timeouts, retries, circuit breakers, etc.). - -![](assets/istio_59b3a1cd6fd186a9203774101e1f6ab6_md5.webp) - -Pilot transforms high-level scheduling rules into Envoy-specific settings that regulate the traffic and propagates them in real time to the sidecars. Pilot summarizes and synthesizes platform-specific service discovery processes (Kubernetes, Consul, etc) into a normal file that can be consumed by any sidecar compliant with the APIs of the Envoy data plane. This loose coupling allows Istio to run on multiple environments while maintaining the same operator interface for traffic management. - -### Mixer - -Mixer enforces access control and utilization strategies across the system mesh and gathers information from the Envoy Proxy and other facilities for telemetry. In another word, Mixer is the monitoring agent of Istio network. - -![](assets/istio_76c3d8d9b9ee843b5445a06359a73111_md5.webp) - -![](assets/istio_b26506ec8999334276375c45f5510191_md5.webp) - -Mixer involves a versatile plugin system. It allows Istio to interact with multiple backend infrastructures. Istio therefore extracts from these information of the Envoy Proxy and Istio-managed facilities. - -### Citadel - -Citadel enables strong service-to-service and end-user authentication with built-in identity and credential management. Citadel can be used to upgrade unencrypted traffic in the service mesh. - -![](assets/istio_44d2292a4e7d6391c544bfb68ad30f41_md5.webp) - -### Galley - -Galley is Istio’s configuration validation, ingestion, processing and distribution component. It is responsible for insulating the rest of the Istio components from the details of obtaining user configuration from the underlying platform (e.g. Kubernetes). - -## References - -- Book: Istio In Action - Christian Posta -- Istio docs: [https://istio.io/docs/concepts/what-is-istio/](https://istio.io/docs/concepts/what-is-istio/) -- [https://medium.com/microservices-in-practice/service-mesh-vs-api-gateway-a6d814b9bf56](https://medium.com/microservices-in-practice/service-mesh-vs-api-gateway-a6d814b9bf56) diff --git a/topics/engineering/pomodoro-technique.md b/topics/engineering/pomodoro-technique.md deleted file mode 100644 index 3984088d2..000000000 --- a/topics/engineering/pomodoro-technique.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: Pomodoro technique -description: Learn how the Pomodoro Technique boosts productivity by breaking work into focused 25-minute intervals with short breaks, helping you manage time and reduce mental fatigue effectively. -date: 2022-09-25 -github_id: namtrhg -redirect: - - /8xnxpA ---- - -## What is Pomodoro Technique? - -The Pomodoro Technique is a time-management philosophy that aims to give the user maximum focus and creative freshness, enabling them to finish projects more quickly and with less mental exhaustion. - -It is an easy process. You allocate time for each project throughout the day in little bursts and take breaks as needed. 25 minutes of labor are followed by a 5-minute rest. - -A "pomodoro" is a 25-minute work block, so named after the Italian word for tomato. The method's name comes from Francesco Cirillo, who utilized a kitchen timer in the form of a tomato as his personal timer. You take a 15-20 minute break after four "pomodoros" (100 minutes of work time and 15 minutes of break time) have been completed. - -You mark your progress with a "X" after each completed pomodoro and keep track of how many times throughout each 25-minute period you were tempted to delay or switch to working on anything else. - -## The technique steps - -The original technique has six steps: - -1. Decide on the task to be done. -2. Set the pomodoro timer (typically for 25 minutes). -3. Work on the task. -4. End work when the timer rings and take a short break (typically 5–10 minutes). -5. If you have finished fewer than three pomodoros, go back to Step 2 and repeat until you go through all three pomodoros. -6. After three pomodoros are done, take the fourth pomodoro and then take a long break (typically 20 to 30 minutes). Once the long break is finished, return to step 2. - -You can also do it the canonical way: - -1. Break down complex projects. If a task requires more than four pomodoros, it needs to be divided into smaller, actionable steps. Sticking to this rule will help ensure you make clear progress on your projects. -2. Small tasks go together. Any tasks that will take less than one Pomodoro should be combined with other simple tasks. For example, "write rent check," "set vet appointment," and "read Pomodoro article" could go together in one session. -3. Once a pomodoro is set, it must ring. The pomodoro is an indivisible unit of time and can not be broken, especially not to check incoming emails, team chats, or text messages. Any ideas, tasks, or requests that come up should be taken note of to come back to later. A digital task manager like Todoist is a great place for these, but pen and paper will do too. - -## Why is it useful? - -The arbitrary silliness of using a tomato as a stand-in for units of time belies the Pomodoro Technique's serious effectiveness when it comes to helping people get things done. - -The pomodoro approach urges you to do precisely that: break down your huge activities, projects, or ambitions into something you only have to perform for the next 25 minutes. It keeps you focused on the next task at hand rather than becoming overwhelmed by the vastness of what you're undertaking. Don't be concerned about the outcome; instead, take it one pomodoro at a time. - -## Reference - -- Wikipedia Pomodoro_Technique [Link](https://en.wikipedia.org/wiki/Pomodoro_Technique). diff --git a/topics/engineering/radix-sort.md b/topics/engineering/radix-sort.md deleted file mode 100644 index 68cc7fff5..000000000 --- a/topics/engineering/radix-sort.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Radix sort -description: Radix sort is a non-comparative sorting algorithm that sorts integers by processing individual digits. Unlike comparison-based algorithms (like Quick Sort or Merge Sort), Radix sort groups numbers by their individual digits. -date: 2024-06-27 -authors: - - vdhieu -tags: - - algorithms - - sorting -redirect: - - /r15sJg ---- - -> Radix sort is a non-comparative sorting algorithm that sorts integers by processing individual digits. Unlike comparison-based algorithms (like Quick Sort or Merge Sort), Radix sort groups numbers by their individual digits. - -## Key concepts - -- **Digit positioning**: Radix sort processes digits from the `least significant digit (LSD)` to the `most significant digit (MSD)`. -- **Stable sorting**: It maintains the relative order of records with equal keys. -- **Counting sort as a subroutine**: Radix sort often uses Counting Sort for sorting digits, ensuring stable sorting at each digit level. - -## Steps of Radix sort - -- **Determine the maximum number of digits**: Find the maximum number in the array to understand the number of digits. -- **Sorting by each digit**: Use `Counting Sort` to sort based on each digit's place value - - Initialize Buckets: create an array of buckets for each digit from 0 to the radix minus one (e.g., for decimal numbers, you need 10 buckets). - - Distribute the Numbers: start with the LSD and move towards the MSD. distribute each number into the corresponding bucket based on the current digit - - Collect Numbers: collect numbers from the buckets and update the list in the new order -- **Repeat for each digit position**: Continue the process for each digit until all positions are sorted. - -![radix sort example](assets/radix-sort.gif) - -## Big O notation - -If we take - -- `n` is the number of elements -- `k` is the number of digits in the largest number - -the time complexity for Radix sort will be `O(n×k)` and the space complexity will be `O(n+k)` - -## Advantages - -- **Efficiency**: Linear time complexity O(n×k) when k is the number of digits. -- **Predictable performance**: Performs consistently regardless of the input data's initial order. - -## Disadvantages - -- **Limited scope**: Primarily useful for integers or fixed-length strings. -- **Memory usage**: Requires additional memory for the Counting Sort process. diff --git a/topics/engineering/reusability-in-software-development.md b/topics/engineering/reusability-in-software-development.md deleted file mode 100644 index 8299b312a..000000000 --- a/topics/engineering/reusability-in-software-development.md +++ /dev/null @@ -1,173 +0,0 @@ ---- -title: Reusability in software development -description: Learn how software reuse strategies, including component reuse, frameworks, and product lines, speed development, cut costs, and improve quality while facing challenges like maintenance and tool support. -date: 2020-05-05 -tags: - - engineering -redirect: - - /S0Kodw ---- - -## Introduction - -In the previous century, when the software market was also **immature**, the economy was grown with a large scale lead to more and more new business problems appear that demands software to solve such as management, automatic. So a lot of new ideas about software were imagined and implement from the roots to resolve the corresponding matter. Because of this original, the reuse of software was uncommon. - -Nowadays, after a long time for deeping in building something new to solve classic problems. The seniors in this software development field have concluded an engineering strategy that is called **Software reuse** where the development process is geared to reuse existing software. The move to reuse-based development has been in response to demand for lower software production and maintenance costs, faster delivery of systems, and increased software quality. There are also matters that are needed to solve when the answer to classic problems was completed increasingly. - -So this strategy is now used extensively in the development of new business systems and the companies are promoting reuse of existing systems to increase their return on software investments. - -There are many kinds of field Software reuse that is being available. The Open-source movement is a representation of reuse where a lot of code that is reuse like libraries or a part of other systems. - -> Open-source: "Open source is a term that originally referred to open source software (OSS). Open-source software is code that is designed to be publicly accessible—anyone can see, modify, and distribute the code as they see fit." - -> Open-source movement: "The open-source movement is based on a radical retake on copyright law to create high-quality software whose use and development are guaranteed to the public." - -Another is the domain-specific application systems, such as ERP systems, are available that can be tailored and adapted to customer requirements. Some big companies have supplied the components that have the ability to be configured to demand each their specific customer. - -Standards, such as web service standards, have made it easier to develop software services and reuse them across a range of applications. - -Reuse-based software engineering is an approach to development that tries to maximize the reuse of existing software. The software units that are reused may be of radically different sizes. For example: - -- Component reuse: Component is a scalable concept, it can be big like a subsystem or small as a class, function, or object and have the ability to reuse flexibly. -- Application reuse: An application can be reused by integrating without change into a system or re-configured for a specific customer. -- System reuse: A system that contains a number of application can be a part of another bigger system. - -Each function or component that includes generic functionality is potentially reusable. However, sometimes, it is very expensive to modify them for a new situation. So, rather than reuse code, the idea is also a good thing to reuse. This way is called concept reuse. - -In concept reuse, instead of component, you reuse ideas, working style, or algorithm. On the other hand, it means everything that is reused is represented in an abstract notation, which does not have an implementation detail. It can, therefore, be configured and adapted for a range of situations. A few methods that depend on concept reuse are design patterns, configurable system products, or program generator. The concept reuse process must contain an activity where the abstract concept is instantiated to create executable components. - -## Few aspects of software reuse - -### The benefits - -The first thing that everyone thinks about the reuse, is the fast development speed when applying another existing component in our system. The reuse provides the ability to bring an application or system to the market as early as possible because both development and validation time may be reduced. It is very helpful when overall development cost is not more priority than delivery speed. - -To the specialists, instead of doing the same work over and over again, they often develop reusable software that encapsulates their knowledge. It is very convenient for development and sharing source code. - -Another advantage of the reuse in software development is available dependability. As you know, each reuse component, application, or system should be stable. It is the result of the long term tried and fixed the problems of the development team, and applied to the working system, or itself is well-working stuff. Almost all its design and implementation faults should have been found and fixed. - -In the first paragraph, I mentioned the reuse as a good option when bringing a system to market as early as possible is often more important than overall development costs. But it is not mean this a waste strategy, vice versa, in another aspect, it makes development costs reduced. This advantage can be explained that development costs are proportional to the size of the software being developed and reuse help development team write fewer lines of code. - -Apply the strategy software reuse also helps the development team reduced process risk because the cost of reuse component is already known, besides, the development cost is always intransparent. So, using software reuse is an effective method for project management by helping the development team reduces the margin of error in project cost estimation. This is especially true when large software components such as subsystems are reused. - -A special point that is not noted, is a lot of standards being used in each engineer's routine is the software reuse, too. For example, when using user interface standards, component as the menu is implemented by using reusable components, all applications present the same menu formats to users. So users will make fewer mistakes when interacting with a familiar interface and application's dependability will raise. - -There is a lot of benefits when using Software reuse in development, but I think the above reasons are convincing enough for us to think about applying the reuse strategy in our software development process. - -Summary, basically, we have six benefits of Software reuse: - -- Accelerated development -- Effective use of specialists -- Increased dependability -- Lower development costs -- Reduced process risk -- Standards compliance - -### The problems - -Besides a lot of benefits, every tool has its own matter, Software reuse is no exception. - -To the development team, it is a hard challenge to build a reusable component library that is favored by other software developers and ensure that the library is used. After building, maintaining this library is also a complex process that contains a few factors such as compatibility, comfortable, and easy to use. - -On the user side of that reusable component, it takes a lot of time to find out a suitable library for their project. The rest is more and more time to understand and adapt this library in a new environment. To the engineer, they have to apply a new one into their development process uncomfortably. - -The above matters are obvious and popular in the real life of every engineer. So I want to mention others with my experience. - -In my recent project, I was required for updating the language version and its own dependencies. Almost all it that is open source projects, I can read line by line of the source code, but I have still had difficulty in doing the update, ensure any changes do not make code break. Have you ever wonder that if source code is not available, how difficult is maintaining the reuse with code broken ability. Yes, it is a difficult process, the reason why maintenance costs will increase. - -Another bad stuff of reuse in software development is the lack of tool support. Some software tools do not support development with reuse. If it only makes difficulty, you can consider between the tradeoff and corresponding benefits. So, if it is impossible to integrate these tools with a component library system, in some situations, you will be obstructive. - -The last one is called **“Not-invented-here” syndrome**, when you focus on cloning, rewriting, optimizing component with the belief that you will make it greater instead of trying to do another solution. This is partly to do with trust and partly to do with the fact that writing original software is seen as more challenging than reusing other people’s software. - -### The reuse landscape - -After time passed, the reuse in software development has been increasingly supported by a lot of new techniques. The base of these techniques is the fact that the system in the same application domain are similar and have the potential for reuse. There are many different ways of Software reuse, from simple components such as class, object to complete system, and that standards for reusable components facilitate reuse. You can see an overall picture of the “reuse landscape”—different ways of implementing software reuse below. - -![](assets/reusability-in-software-development_ea401e3ee43cf4ee90e7edc92fe83900_md5.webp) - -if you feel unclear and want to walk into details of each approach of Software reuse, you can see the following figure. - -![](assets/reusability-in-software-development_54089293152bc5fd40b4e0ddb7dd69e7_md5.webp) - -After having an overview of the reuse landscape, for sure, you will wonder “which is the most appropriate technique to use in a particular situation?”. - -The answer to the above question depends on a lot of things such as system requirements, technology and available reusable assert, and the expertise of the development team. But there are a few key factors that you should consider when planning reuse will be mention below. - -1. The development schedule for the software: In the context that you want to bring your system to the market as early as possible, reuse complete systems should be your choice instead of individual components. Although the system will not be fit with the requirement perfectly, this approach minimizes the amount of development required. -2. The expected software lifetime: Maintainability and scalability are the highest prioritizations when you develop a long-lifetime system. So choosing a reuse component whose source code can't be accessed is not wise. A personal component or open-source system can be a good solution for this situation. With the ability to access source code, you don't worry when suppliers may not be able to continue support for the reuse software. -3. The background, skills, and experience of the development team: In before [section](https://github.com/dwarvesf/radar/blob/master/software-reuse/Documents/software-reuse.md#22-the-problems), it takes a lot of time to find out a suitable library for their project. The rest is more and more time to understand and adapt this library in a new environment. Therefore, you should focus your reuse effort in areas where your development team has expertise. -4. The criticality of the software and its non-functional requirements: For a critical system that has to be certified by an external regulator, you may have to create a safety or security case for the system. This is difficult if you don’t have access to the source code of the software. If your software has stringent performance requirements, it may be impossible to use strategies such as model-driven engineering (MDE). -5. The application domain: In the market, many domains that require the same features for almost all their application such as manufacturing and medical information systems. So we can reconfigure an existed application to use in our place at a cheap cost instead of developing a new system. -6. The platform on which the system will run: Some components models are developed to using in a specified system such as .NET in Microsoft platform. The rest is the generic application that can be used in the multi-platforms. Every engineer needs to choose a reuse system that fits with development process designed. - -Above is a few stuff that building and groundwork for making a decision for the questions such as when we need Software reuse, or what is the best solution for our project. Where or not reuse is applied, is often decision by manager instead of engineer. Sometime, they evaluate the risks within their choice incorrectly. Others may prefer known risks of development to unknown risks of reuse. So i think if you want to perform a decision making, you must have all of your solution on the table, the more you understand your solutions the more accuracy your decisions are. In this context, this is reuse-related decision. - -## Application framework - -Before explore about Application framework, let's revise object-oriented development. Following is an difinition of OOD that i see on [Quora](https://www.quora.com/). I think this is really good sentence for mentioning to it - -> Object-oriented Development (OOD) a group of methodologies that sees real world entities as objects and classes. For example, hospital is a real world entity, becomes hospital class and later multiple hospital objects are created, each with unique property values. - -For a long time, some enthusiasts for object-oriented development suggested that one of the key advantages when using an object-oriented approach is reusing previous work. That means you can use the same object for different systems. However, to me, in my work, when coding and developing a system, I see a truth is that we need specified objects or classes for a particular component or application. Another bad thing is that we often spent more time to understand and adapt the object than reimplement it. - -So, instead of using OOD directly, we have an object-oriented development process that is the best support for object-oriented reuse through larger-grain abstractions called frameworks. - -> Framework is an integrated set of software artifacts (such as classes, objects and components) that collaborate to provide a reusable architecture for a family of related applications. - -The first characteristic that is bounced off when I think about the framework is framework provides support for generic features of the domain that is focused on this framework. For example, in my primary programing language - [Golang](https://github.com/dwarvesf/radar/blob/master/software-reuse/Documents), it has a web framework being named [Echo](https://github.com/dwarvesf/radar/blob/master/software-reuse/Documents) - a high performance, extensible, minimalist web framework for Go. Besides a lot of salient features, it has generic features of a web framework such as routing, database integration, authentication, data rendering. - -Another example is [Unity](https://github.com/dwarvesf/radar/blob/master/software-reuse/Documents) that is known as a game engine or game framework, it also has a lot of generic features that always exists on each game engine such as audio system, graphic, animation, UI widgets or hardware event handling... Depend on their features, tools, each engineer can comfortingly creative. For web engineers, this is extending APIs, integrating the third party, implementing new web services by using existing functions for method. For game developer, it also drags and drops UI object to defines game scene layouts or handle mouse event to make an NPC (non-player character) does few animations. More and more. - -As you see, the framework helps us have a lot of things to reuse. It can ether skeleton architecture for building from bottom to top of a system or only a method that is used for integration of an external component. Although existing in different scales, but framework components always support each other or being reuse to construct another bigger. It is easy to see that the architecture is implemented by the object classes and their interactions. Classes are reused directly and maybe extended using features such as inheritance and polymorphism. - -Another fact is a framework can contain others. It begins from common sense that a framework can big or small and can be used to implement a complete system or also a small part of an application. For example, in Unity, the above stuff that I called a framework, has a lot of nested frameworks such as Unity2D, Unity3D, and a lot of external stuff for doing specific things. It can be an action Framework for increased performance and ease of development, game object distance/time weighting framework, or a full 2D runner framework and game sample. - -Finally, we can consider a few class of framework. - -I saw a few types of framework in [Sommerville software engineering 10th edition](https://dinus.ac.id/repository/docs/ajar/Sommerville-Software-Engineering-10ed.pdf), see the following: - -- System infrastructure frameworks: tools for doing everything in infrastructures layer such as communications, user interfaces, and compilers. -- Middleware integration frameworks: line of frameworks that support construct an ideal environment for an application works. For example, there are Microsoft’s .NET and Enterprise Java Beans (EJB). What is the ideal environment? This is a location that allows each component of your system to communicate with each other and with OS and exchange data easily. -- Enterprise application frameworks: specified frameworks for the specified domains, such as telecommunications or financial systems. It doesn't only support the development, but also contains this domain's deep knowledge. This kind of framework is the key of the product line that is considered in the next section. - -To me, I prefer another way to classify frameworks that looks more specific and practical. - -- Web Application framework -- Application framework -- Multimedia Framework -- Game Framework -- More and more, I don't think it's mandatory to naming as long as you feel comfortable and understand what is under the hood. - -Summary, framework is used for the reuse purpose, so, an application is constructed base on frameworks that can be reused, too, and the product line is a representation of this application, will be considered later. - -Be a very effective approach to reuse but the framework approach also has corresponding disadvantages. Introducing a framework for another engineer or development team is expensive. I saw a lot of good framework being spanked until someone digs it up randomly. On the development team, it is difficult to approach a new framework and hard to debug if this framework's source code is not available. - -## Software product lines - -In real life, when going to the hospital, we see the medical management application on the doctor's desktop similarly. In the cafeteria, store management applications with the same features all time. Have you ever wonder why are they always like that? In short, these are the representations of product lines. - -What is product line? - -As above, we can understand that is the application when a company wants to serve a few kinds of customers that have their own characteristics, or a software development team is required to create an application with similar features for different customers. Instead of developing software for each case, the development team can create one that has the ability to being configured to suit unidentical requirements. On the other hand, this is a system with a common architecture in core and shared components with a few features that is can be configured flexibly. That configuration can relate to the configuration of some components, implementing additional components, and modifying some of the components to reflect new requirements. - -Generally, product line derives from an exist application that is called base application of product line. Base application is usually designed to simplify reuse and reconfiguration. Generally, a base application includes three kinds of component as following image: - -[The organization of a base system for a product line](https://github.com/dwarvesf/radar/blob/master/software-reuse/Documents/software-reuse.md#3-4-7-8-%22software-engineering%22-httpsdinusacidrepositorydocsajarsommerville-software-engineering-10edpdf) - -- Core components that provide infrastructure support. These are often immutable when developing other instances of the product line. -- Configurable components that may be modified and configured to specialize them in a new application. Sometimes it is possible to reconfigure these components without changing their code by using a built-in component configuration language. -- Specialized, domain-specific components some or all of which may be replaced when a new instance of a product line is created. - -On the other hand, we can distinguish the components of the product line into 4 layers that are as following: - -1. Interaction layer: User interface is located here. Users can visualize intuitive information being sent from the system and interact backward. -2. I/O management layer: handle event or signal with information from the user interface, validate and preprocess data such as authenticate, map output, route planning, provide a mechanism to communicate with the lower layer. -3. Resource management layer: core logic that handles data, process, the query to the database and normalizes data to render to user. -4. Database management layer: build database's components and this related logic such as database function, trigger... - -Besides, depending on these component's type, we have various types of specialization of a software product line may be developed: - -1. Platform specialization: Depend on the platforms where the system is built on such as MacOS, Linux, or Windows, we have corresponding versions of the application in the product line. In order to do that, the component that interfaces with the hardware and operating system is modified. -2. Environment specialization: This environment is the operating environment. It can be peripheral devices or communication environments of different hardware in the system. So we need different versions of the application to fit the environment. -3. Functional specialization: This a more popular specialization in the product line. You can see every banking application of different banks with a lot of generic features, but in a bank, you need at least 50$ to do a transaction, another can be 150$. -4. Process specialization: Make an application of product line fit with specific business processes. diff --git a/topics/engineering/skill-of-software-engineer.md b/topics/engineering/skill-of-software-engineer.md deleted file mode 100644 index fa5195fc6..000000000 --- a/topics/engineering/skill-of-software-engineer.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Skill of software engineer -description: Discover essential software development skills including coding, debugging, version control, design, and automated testing, plus key soft skills like teamwork and communication for effective collaboration. -date: 2017-03-14 -tags: - - engineering -redirect: - - /e46fQA ---- - -1. **Read code** - -The ability to understand an existing code base in order to analyze its behavior and make fixes or enhancements to it. - -1. **Write code** - -This does not include any significant amount of design – just the basics of coding. An example is being able to write a method given the desired behavior (inputs, outputs, pre-conditions and post-conditions). - -1. **Use of integrated development environments** - -The ability to effectively use modern IDEs and an understanding of their strengths and weaknesses. - -1. **Use of version control** - -This skill includes basic use of a version control tool as well as a general understanding of software configuration management. - -1. **Use of libraries and frameworks** - -This skill could also be called "Reuse Existing Code". This skill includes the ability to search for and evaluate libraries and frameworks based on how effectively they meet your needs and the ability to integrate the chosen package into the software you are writing. - -1. **Debugging** - -The ability to analyze the behavior of code to diagnose a problem and find the underlying cause. This includes but is not limited to using a debugger. - -1. **Research and self learning** - -The ability to upgrade level and get more knowledge by yourself - -1. **Design software** - -The ability to determine what code is necessary to achieve some specified functionality, particularly the higher-level structure or organization of the code. - -1. **Awareness of the software development life cycle** - -This is an awareness of the 'big picture' of software development beyond just writing code - how the other life cycle stages (requirements, design, testing, and maintenance) impact coding and vice-versa. This includes an understanding of the types of methodologies (e.g. Agile or Waterfall) that can be used to progress through this cycle. - -1. **Automated unit testing** - -This is the ability to unit test code by writing automated tests. - -1. **Refactoring** - -The ability to revise existing code without impacting its functional behavior. - -Beside that, please also pay attention to the soft skill: Teamwork, Management and Communication diff --git a/topics/engineering/software-development-life-cycle-101.md b/topics/engineering/software-development-life-cycle-101.md deleted file mode 100644 index 6ca95f853..000000000 --- a/topics/engineering/software-development-life-cycle-101.md +++ /dev/null @@ -1,173 +0,0 @@ ---- -title: Software development life cycle 101 -description: Learn the Software development life cycle (SDLC) basics, key project phases, and essential artifacts to build high-quality software on time and within budget for successful project delivery. -date: 2020-05-19 -tags: - - sdlc -redirect: - - /vuQbRQ ---- - -This workshop contains the basic knowledge on Software development life cycle, provides people with a step-to-step guideline and the artifacts which will be created on the way. We don't dive in the details. Instead, we treat it as an overview look on how to build software successfully. - -## What is Software Project? - -We build software, and we need a planned undertaking. So call software project is “A specific plan or design” or “A planned undertaking”. - -## Project constraints - -A software project has a lot of constraints. Cost, scope, quality, customer satisfaction, risk, resource, time, or anything in between. - -But the most important ones are - -- Quality -- Budget -- Time - -![](assets/software-development-life-cycle-101_8d20afb24ee3dfc8607352f6574e549a_md5.webp) - -## Why does a project fail - -With a lot of constraints, the project is easy to fail. We could have plenty of reasons why a software project fails: team politics, overdue payment,... but three of them could be prevented easily with proper methodology, framework - -- Unclear/misleading project requirements -- Wrongly defined tech stacks -- The wrong approach, develop practices - -## Project success - -If it's easy to fail, then what is a successful project? - -> The project is complete on time, on budget and have low defects (high quality) - -This is just a simple definition of what is a successful project based on 3 important constrains. In the end, we want to build software that "awesome" within time and budget. - -But how? - -## Software development life cycle - -The secret sauce of a successful project lies in the answer to those questions: - -- How to perform the step? -- Who is responsible for doing the step? -- Which artifacts will it produce? -- How long will it take? -- Which step should we do next? - -So which steps are we talking about? -We're talking about the Software development life cycle, which sounds familiar to all CS, CE folks. -For the rest of us who take a nap in class-time or for those who are new in the field, Software development life cycle (SDLC) refers to a methodology with clearly defined processes for producing software with the highest quality and lowest cost in the shortest time possible. -SDLC provides a well-structured flow of phases that help an organization to quickly produce high-quality software which is well-tested and ready for production use. - -In detail, the SDLC methodology focuses on the following phases of software development: - -- Requirement analysis -- Planning -- System design -- Implementation -- Testing -- Deployment -- (Maintenance) - -Let's take a look back on what we have learned and how this methodology could guarantee project success. - -![](assets/software-development-life-cycle-101_c9f99108433a5d2449ef51fb884a23dd_md5.webp) - -### Requirement analysis - -“What are the current problems? What are we gonna build?” This stage of the SDLC means getting input from all stakeholders, including customers, salespeople, industry experts, and programmers. Learn the strengths and weaknesses of the current system with improvement as the goal. -Business-oriented is a key in this stage. There are plenty of technique being used **(Lean canvas, AARRR Framework, Industry Research, User Research, Competitor Analysis, Personas, Problem Statement, User Journey Mapping)** and with a lot of deliverables to analyze requirements and validate the business model that the software aims to empower. - -The two most important artifacts of this stage that need to be well-documented is - -- **Lean canvas:** a lean business model which defines problem - solution - market - cost - revenue of a digital product. Lean canvas needs to be validated before moving to the next stage else we will develop software based on imaginary - -![](assets/software-development-life-cycle-101_3a385622cfd9745a70201ab14b616a09_md5.webp) - -- **AARRR funnel**: after the Lean canvas is validated, define funnel for each revenue stream. This artifact will be the foundation of the System design Stage. - -![](assets/software-development-life-cycle-101_fdea8ce098bbc365ecd7ffaaec9010a4_md5.webp) - -Those two will be generated with the agreement between **Product Managers, UX researchers, and Clients.** This stage would take time, but with well-documented artifacts in-hand, we could save a lot of time later on. - -### Planning - -We need a plan (obviously) after the requirement analysis phase complete. - -“What do we want?” In this stage of the SDLC, the team determines the required cost and resources for implementing the analyzed requirements. It also details the risks involved and provides sub-plans for softening those risks. -In other words, the team should determine the feasibility of the project and how they can implement the project successfully with the lowest risk in mind. - -> #Why: To manage project constraints: (or we will fail real damn fast) - -![](assets/software-development-life-cycle-101_40fe66c43cf65cacbed33cf70df11995_md5.webp) - -We have a validated business model, few funnels of revenue streams. We have things that need to be built. Now we need a plan to build it at top-quality within budget and time. - -Product Manager and Technical Architecture need to sit down with Clients to define some sweet things - -- **Project charter** (Product Roadmap, Milestone Release, People in charge...) - -![](assets/software-development-life-cycle-101_2c5adf742029b611a1459c48c5d1ccb8_md5.webp) - -- **Work breakdown structure** (Job need to be done) -- **Tech stacks** - -Some minor things will be defined at this stage as well such as Communication channel, Tooling..., etc. - -### System design - -Based on the produced artifacts (AARRR funnel, Project scope, Product roadmap...), the foundation of the system will be built at this stage. - -“How will we get what we want?” This phase starts by turning the software specifications into a design plan called the Design Specification. All stakeholders then review this plan and offer feedback and suggestions. It’s crucial to have a plan for collecting and incorporating stakeholder input into this document. Failure at this stage will almost certainly result in cost overruns at best and the total collapse of the project at worst. - -The list below is just as general as possible but those artifacts are the least we could have after finishing System design stage - -**Information Architecture Design** (IA): The foundation of information which will be presented to connect the user to the content they're looking for when using the software. - -![](assets/software-development-life-cycle-101_121f206d154f23e1049f0edd39e921cf_md5.webp) - -**Software Modeling** (Usecase Diagram, State Machine Diagram, Activity Diagram, High-level Architecture Diagram, ERD...) - -- User flows, User stories, Wireframe -- Design System -- User Interface, User Interaction Design - -The roles - -- UX Designer -- Technical Architecture (Software Engineer) -- Visual Designer - -### Implementation - -> #“Let’s create what we want.” - -At this stage, the actual development starts. Every developer must stick to the agreed blueprint. Also, make sure you have proper guidelines in place about the code style and practices. - -This is the longest stage of SDLC Process. Work is divided into units or modules and assigned to various Software Engineering. The **project quality** is set by this phase. There're knowledge areas that Software Engineer could dig deeply to improve and understand Software Craftsmanship. - -Make it count and share with the team what you have learned recently. - -![](assets/software-development-life-cycle-101_c5d490b4e34e3a156230e0f702348538_md5.webp) - -### Testing - -“Did we get what we want?” In this stage, we test for defects and deficiencies. We fix those issues until the product meets the original specifications. - -In short, we want to verify if the code meets the defined requirements. -Provide stakeholders information about the **project quality** then sign off application deliverable to release it to end-user. - -![](assets/software-development-life-cycle-101_fce9cd7be98a3a0133ae89129f323211_md5.webp) - -### Deployment - -> Let’s start using what we got. - -Now, the goal is to deploy the software to the production environment so users can start using the product. However, many organizations choose to move the product through different deployment environments such as a testing or staging environment. - -This allows any stakeholders to safely play with the product before releasing it to the market. Besides, this allows any final mistakes to be caught before releasing the product. - -Job needs to be done - -- Setup infrastructure (server, domain, database, ...) -- Automation process (CI/CD) diff --git a/topics/engineering/software-modeling.md b/topics/engineering/software-modeling.md deleted file mode 100644 index 927622bb2..000000000 --- a/topics/engineering/software-modeling.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -title: Software modeling -description: Learn how software modeling improves system maintainability and scalability by defining core objects, using diagrams like ERD, SMD, and USD to design clear, efficient applications. -date: 2020-05-08 -tags: - - modeling -redirect: - - /fhLz3Q ---- - -This is how I approach a new topic or knowledge, starts with why, understand the concept and figure out how to make it happen. - -## Why do we need Software modeling - -This can be summed up into 2 definitions: - -- Maintainability: Software modeling helps to maintain a system, or a project. It ensures the based document for newbies to approach and get to know the system thoroughly. -- Scalability: helps developers to add sub-system or new feature onto the current one, without creating conflicts. - -## What is Software modeling - -According to Wikipedia, Software modeling came out in a lengthy definition. So I took the privilege to paraphrase it like below: - -→ **Software modeling is how to turn an activity of an object from reality to a form that computers can understand and execute.** - -## How to do Software modeling - -People create things to help them solve a problem, instead of doing that themselves. Tools are made to help us do the work easier. The purpose of Software modeling is to reconstruct the actions that an application needs to take, or the product owner wants to happen. - -### Action - -I'll take the marketplace as an example of Action in the business world. A marketplace, such as Joolux & Purchasing Care, is a platform where buyer and seller exchanges the goods. An action in marketplace can be defined as: - -→ Buyer buys Product from Seller - -### Model of a marketplace system - -After defining an action, we need to break it down and find the core objects. In this case: - -→ Core Object: **Buyer**-**Product**-**Seller** - -### AARRR framework - -In the business world, AARRR is a habitual method. AARRR is a funnel to analyze the business growth and business development orientation of a company. - -Our Design team is using this to follow the final goal of an application or a system. By applying AARRR, designers won't get lost along the way, or happen to conduct redundant things. The action of an application/ system will go through the end of the AARRR funnel, which lets us know how that action derives revenue stream. - -**Huy Nguyen** : So we need to define the core object before or after applying the framework? Is there any way to locate the core object without using AARRR? -**Khiem Vo**: Normally, an application comes with a landing page. That page displays a tagline, or a short description on the app's purpose. For instance, take a look at [sudo.fm](https://sudo.fm/). We get to see a song name, a menu bar for sound control and basic function of a music site. That's how we know the main action of [sudo.fm](https://sudo.fm/) is a music player. -**Huy Nguyen**: So before identifying the core object, we should spend some time understand the application. But what if it's a new project with no product or landing page? -**Khiem Vo**: For that case, we'll adapt the waterfall structure. It means to collect the requirement. What's the idea of that app? What problem is it trying to solve? -**Huy Nguyen**: And Design team will conduct that part? -**Khiem Vo**: Personally I don't think that work belongs to a specific team. I'd prefer both Designers and Developers to understand the requirement and come up with a direction. Software modeling is how we get there, by analyze the system requirement and what should we do to make it happen. - -After that, we'll find out the relationship between them. - -### Entity Relationship Diagram (ERD) - -A relationship between core objects is demonstrated in a form of an entity relationship diagram (ERD). - -ERD helps to show - -- Main object: the core object in the scope system -- The relationship between the objects - -Components of ERD - -- Object -- Relation arrow - -An ERD should have these things to keep in mind - -- We don't need to re-draw the database in object field. The focus point of ERD is to show the core object and the connection between them. -- Coloring. Using color will categorize the objects and visualize the system construction. -- Arrange the objects based on the pipeline of the main action - -ERD helps simplify the database, giving us the first impression on a system, what kind of object, how many table and the relationship between them. -To create an action, we need to add state - the status of how core object will change during the process of an action. - -## State Machine Diagram (SMD) - -Purpose - -- Demonstrates the status that an object will walk through -- Demonstrates the response of objects during the process - -Components - -- State: The hardest part. The definition of the object's status. -- Action: What makes an object to change the status -- Actor: Who will execute that action - -To create a completed main action, every object needs its own SMD. The core of a SMD is to define the correct state of an object, and whether or not if that state is related to the system. There are some state which the system doesn't cover. Based on the requirement and the business scope, we will decide which state is necessary. - -Important Notes - -- Display the actor and the action -- A state must be an adjective - -**Giang Vu**: I notice you've mentioned the different state between the real world and in SMD system. Do you have any example for that? -**Khiem Vo**: Sure. Let's look at a marketplace for grocery. The object will be the products. Grocery is perishable goods, which means "rotten" or "fermented" is also a state. But we don't need to list that into the SMD, because it's unrelated. We only do it if the states are "shipped", "packed" or "returned". -**Giang Vu**: So that means the state will need to be business-oriented? -**Khiem Vo**: True. Another thing to remember is we only have to list out the states if it's a part of the system flow. - -### Use Cases Diagram (USD) - -Definition - -- A form of system requirement -- To help design a system from end user's perspective, allowing developers/designers tp walk in the clients's shoes - -Components - -- Actor: End users, people who will use the system -- Use case: The system function -- Boundary system: Letting designers/ developers know their current stage of Software modeling. Boundary system groups the actions into sub-system, makes it more specific than in the ERD - -→ In USD, we don't need to follow any order. The items should be listed out randomly and regroup it into sub-system to create UI or interface. - -**Huy Nguyen**: If two actors are conducting one action, we have to 2 different results. Would it be possible to display it on the same USD? -**Hieu Phan**: Actually each actor will have a separate use case. -**Giang Vu**: Does that mean each boundary system is made for one actor only? -**Khiem Vo**: As I know, boundary system allows us to create different interfaces for different actors. It's about how many scenario can possibly happen. But I'm not sure one actor can involve in more than one interface at a time. - -### Component diagram - -Definition - -- Visualizing: The main components of a system -- Constructing: How the system can be executed - -Components - -- Component: Describe a module of a system -- Provided interface: Represent an interface that the components provide -- Require interface: Represent an interface that the components require - -Through a component diagram, a source code can contain - -- Front-end: How many interface/ main module -- Backend: Module of each object to collect database -- A place to log the arising situation during the process of calling API - -## Recap - -Software modeling in Agile team - -- Helps teammate to possess the same base knowledge about a system -- Helps document the information for team discussion, research and understanding -- Every change can create a big impact on the system. In Agile, a product can be modified continuously, Software modeling needs to be updated during the whole cycle to make sure the newbies can catch up - -When we have the insight and the view of the Product Owner, it's easier to create an effective and outstanding outcome. This also reveals the spirit we've been pursuing - Craftsmanship, by providing client with values from their own perspective. diff --git a/topics/engineering/software-quality-assurance.md b/topics/engineering/software-quality-assurance.md deleted file mode 100644 index 2a8580665..000000000 --- a/topics/engineering/software-quality-assurance.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: Software quality assurance -description: Learn how Software Quality Assurance ensures software meets requirements through testing, planning, and process control to improve reliability, usability, and maintainability in development projects. -date: 2022-10-04 -github_id: trkhoi -redirect: - - /ruz1fA ---- - -## Definition of software quality assurance - -`Software Quality` is the degree of conformance to explicit or implicit requirements and expectations. This leads to 2 levels of Software Quality: - -- Functional: the product’s compliance with functional (explicit) requirements and design specifications. This aspect focuses on the practical use of software, from the point of view of the user: its features, performance, ease of use, absence of defects. -- Non-functional: system’s inner characteristics and architecture, i.e. structural (implicit) requirements. This includes the code maintainability, understandability, efficiency, and security. - -`Software Quality Assurance` is a process that assures the software product meets and complies with the organization’s specification. It is a set of activities that verifies everyone involved in project implemented correct procedures and processes. - -## Attributes of software quality assurance - -There are some properties that we can based on that to assure quality: - -- Correctness: extent to which a project fulfills its specifications. -- Efficiency: use of resources execution and storage. -- Flexibility: ease of making changes required by changes in the operating environment. -- Integrity: protection of the project from unauthorized access. -- Interoperability: effort required to integrate the system to another system. -- Maintainability: effort required to locate and fix a fault in the project within its operating environment. -- Portability: effort required to transfer a project from one environment to another. -- Reliability: ability not to fail. -- Reusability: ease of re-using software in a different context. -- Testability: ease of testing the project to ensure that it is error-free and meets its specification. -- Usability: ease of use of the software. - -## Why we need software quality assurance - -1. Common problem - -- Feature has many bugs. So who will be responsible for that ? There’s one story about it: QC engineer said: “Hey developer A, did you implement this feature ? It has many bugs”. Developer A said: “No it’s not bug, I just follow the design of Architecture Engineer”. Architecture Engineer said: “The design follows what Project Manager said”. PM said: “No, this is because customer keep changing their requirement”. The story just keep going and it takes us a lot time just to argue. So why this happen ? People usually blame each other and they just care about their job, like developer just care about coding and when requirement changes they will say it’s not their fault. - - So the point here is lack of knowledge → we need to know carefully about software procedure. - -- The relationship between Developer and QA/QC is not good. This is because some developer will be uncomfortable when QC keeps complaining about the bug and requires developer to fix that. This lead to many quarrel. - So the point here is lack of knowledge about QA/QC role. Their role is: - - Use feature and let Developer knows if the result is success or fail. - - They will assure that if Developer follows procedure or not. -- Junior or some inexperienced developer skip some step in software procedure like they don’t carefully ask for the requirement and they don’t design system carefully but start to implement the feature. This will lead to implement wrong and will take a lot of time to fix it. - So the point here is lack of knowledge about software procedure -- Some developer does not test their feature carefully like not write unit test, not test their feature on staging env and release to production, or they have mindset “implement first, fix bug later”, … And when deliver to customer, they won’t accept that because there’s too many bugs. At this time developer needs to fix it and it will take more time than when they implement it. - -2. Principles of Software Quality Assurance: - -From many problems we met in real project, we should know principles of SQA to do our job better: - -- Testing show mistakes: So when QA test feature and feedback to developer. It’s normal there’s no need to quarrel, we should focus on fixing it. -- Early testing: Need to test the feature ASAP from the very beginning like unit test in code, test the feature in local → develop → staging → uat → production. The later we test, the more we pay. The chart below show that. -- Update test: As soon as the errors are fixed, the test scenarios become useless. It is important to review and update test regularly. - -![](assets/software-quality-assurance_errors-cost.webp) - -- Invalid and unexpected test: Need test all case can happen not only happy case or some basic invalid request. -- Independent test environment: Should be no change during the process of testing -- Context dependent: Not every software is tested the same way. For example: fintech software needs correctness so we need test very carefully, but corporate website needs speed and usuability. - -## Role of agile in software quality assurance - -We already know the spirit of Agile Development Lifecycle: `Retro and adapt`. We run sprint, have a short planning for about 2 weeks. When have any issue then we figure it out and discuss what the best option for it. The cycle keeps continue to the final product (this means the product meets client’s requirement, …). This help us figure errors faster and fixed it ASAP. - -![](assets/software-quality-assurance_agile.webp) - -## Process of software quality assurance - -We have known the importance of SQA then we should have effective plan for our software: - -- Define Quality Assurance Plans to identify what to do to ensure quality. - - The Quality plan describes how the management of quality will be applied to the project and confirms any quality standards, procedures, techniques that will be used in the project. - - Quality plan must be develop at the beginning of the project - - Can be a separate plan or a part of the project plan, depending on the types of the project - - Quality must be part of the software development process, not something to be measured at the end. - - Everybody on the project should be responsible for the quality, including project manager, developers, testers, quality assurance -- Support Project Manager to define standards, guidelines and other techniques for the projects -- Ensure systematic quality control of processes and products such as reviews, inspections, audits as well as configuration management/control, production release, project control, supplier contracting and management, document control, operations and support, maintenance, backup and recovery, and security. - - Requirements Review: Testers review the software requirements to understand them and make sure that they are testable - - Test Planning: Testers know what needs to be tested, and plan their testing activities such as: Prepare test strategy, test plan, test schedule and estimate the testing time - - Test Designing: Testers begin to build test cases, test scripts and test data based on the requirements/design of the software - - Test Environment Setup: Testers setup the test environment and make sure that it is the same as the users’ environment. - - Test Execution: Testers execute their test cases and test scripts in the test environment to determine the quality of the software (Pass/Fail) - - Test Reporting: Document the testing logs and status reports. Retrospective meeting will need to define and have solution to solve it. - -![](assets/software-quality-assurance_the-stage-of-software-testing.webp) - -- Maintain quality records to track issues. -- Analyze and report on quality issues to management. -- Maintain and improve quality of products - -## Reference - -- [Blogs of Prof. John Vu, Carnegie Mellon University](https://science-technology.vn/?s=chất+lượng+phần+mềm) -- [Altexsoft whitepapers](https://www.altexsoft.com/whitepapers/quality-assurance-quality-control-and-testing-the-basics-of-software-quality-management/) -- [Blogs of Danlew](https://blog.danlew.net/2022/06/22/maintaining-software-correctness/) diff --git a/topics/engineering/story-point-estimation.md b/topics/engineering/story-point-estimation.md deleted file mode 100644 index d5988f631..000000000 --- a/topics/engineering/story-point-estimation.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Story point estimation -description: Learn how agile teams use story points and collaborative methods like Planning Poker to estimate workloads accurately, improving sprint planning and product development efficiency. -date: 2022-09-15 -github_id: nhuthm -redirect: - - /z9Q1Ew ---- - -From most of the team project, with higher roles like: project manager, product owner, and engineering manager will need to understand the workload of each members from the team, to see what they has been doing so far in the product development. The story point is one of the way to determine that. - -So, at this point, story points estimation is one of the important part when defining the works in the future sprint for each members in the team during the Sprint Planning. A good estimation will help product owners optimize for efficiency and impact. How do we do the estimation precisely? - -## Collaborate with your project manager, product owner - -Usually, before the sprint planning happens, PM/PO will take time to prepare for the tickets are going to do in the future sprint - most of the tickets are captured based on the desired features and fixs for a product from customer's requirements. - -Hence, those tickets are in the very common statement since most of the PM/PO does not understand much from the technical site for the implementation. Leading to hard for all of the members to estimate their work. - -So, to separate works precisely and more easy to estimate, a team members can give PMs new insight into the level of effort for each work item. Like, can giving PMs a definition of their work for example. - -## Agile estimation is a team sport - -During the estimation, it should be involving by eveyone members in the team (developers, designers, testers, etc.). Why? Because each of them will have a different perspective on the product and the work required to deliver for the feature. - -## Estimate smarter, not harder - -Usually, when estimating, we will based on the time that our member need to finish their work. For example: 1-2 points for about 2-4 hours. So when having a big requirement ticket, we should break it into the subtasks or smaller tasks to keep track the work better and easy for the team to estimate also. - -Another common method that most of the team using to estimate calls Planning Poker. For this method, it is definitely requires the participant of every members when doing the estimate. It is simple that, after the providing information for the ticket's story, based on the agreement from every team players, we can have a final estimation for the ticket. For more information, can walk through with the provided documentation from Atlassian team from the link below. - -## Learn from the past estimates - -After every sprint, we have a meeting calls "Retrospectives", each members will giving their opinion about the previous sprint. We will based on to discuss and give the proper solution on the next job for estimating the work better for the team. - -## Reference - -- https://www.atlassian.com/agile/project-management/estimation -- https://www.atlassian.com/blog/platform/a-brief-overview-of-planning-poker diff --git a/topics/engineering/stream-aligned-team.md b/topics/engineering/stream-aligned-team.md deleted file mode 100644 index 889ebd1d6..000000000 --- a/topics/engineering/stream-aligned-team.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: Stream aligned team -description: Learn how stream-aligned teams enable fast delivery by owning end-to-end work, reducing hand-offs, and adapting quickly to change for better software flow and customer feedback. -date: 2022-09-19 -github_id: zlatanpham -redirect: - - /2fZP2w ---- - -A stream is the continuous flow of work aligned to a business domain or organizational capability. A stream requires clear goals and responsibilities so that multiple teams can coexist, each with their own flow of work. - -A stream-aligned team is a team that works on a single stream of valuable work, such as a product or service, a set of features, or a user persona. Such teams typically have complete ownership over the work they do; this means that they do not require hand-offs from other teams to perform any part of their work. - -A stream-aligned team, working on the full spectrum of delivery, is positioned to respond quickly to feedback from customers and to monitor their software in production. They are the primary team type in an organization, and other types of fundamental team topologies exist to support them. - -Generally, stream-aligned teams require capabilities to progress work from its initial exploration stages to production. These capabilities include: - -- Application security -- Commercial and operational viability analysis -- Design and architecture -- Development and coding -- Infrastructure and operability -- Metrics and monitoring -- Product management and ownership -- Testing and quality assurance -- User experience (UX) - -## Expected behaviors - -- A stream-aligned team attempts to deliver a steady stream of features. -- Stream-aligned teams are capable of quickly adapting to changes as they arise. -- A stream-aligned team is experimental and adaptable, allowing it to learn from its mistakes as it goes. -- A stream-aligned team has minimal to no hand-offs of work to other teams. -- Stream-aligned teams are evaluated on the sustainable flow of change they produce, as well as some supporting technical and team-health metrics. -- A stream-aligned team should have time and space to address code quality changes to ensure that changing the code remains safe and easy. -- A stream-aligned team regularly reaches out to the supporting fundamental-topologies teams (complicated subsystem, enabling, and platform). -- A stream-aligned team members feel they are on the path to achieving “autonomy, mastery and purpose.” - -## Reference - -- _Team Topologies: Organizing Business and Technology Teams for Fast Flow by Manuel Pais and Matthew Skelton_ diff --git a/topics/engineering/team-toplogies.md b/topics/engineering/team-toplogies.md deleted file mode 100644 index ef77762f4..000000000 --- a/topics/engineering/team-toplogies.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: Team toplogies -description: Discover how Team Topologies improves software delivery by organizing teams into four types and using three interaction modes to boost collaboration, reduce cognitive load, and enhance flow. -date: 2022-08-15 -github_id: zlatanpham -redirect: - - /qJZZ9g ---- - -Many organizations experience problems with software delivery because they have an unhelpful model of what software development is really about. An obsession with “feature delivery” ignores human-related and team-related dynamics inherent in modern software development, leading to disengagement from staff, especially when there are high cognitive loads. - -The Team Topologies pattern enables teams to address all these points by establishing a [[Team first thinking | team-first approach]] to software delivery based on four fundamental team types, three patterns of interaction between teams, and ways of turning difficulties in delivery into signals for the self-steering organization. - -A software organization can be run with only four team types: - -- **[[Stream-aligned team | Stream aligned]]**: a team that is aligned with the main flow of business change, with cross-functional skills and the ability to deliver significant increments, can move forward without waiting on another team.. -- **Platform**: a team that develops the underlying platform supporting stream-aligned teams in delivery, thereby simplifying otherwise complex technology and reducing cognitive load for teams that use it. -- **[[Enabling team | Enabling]]**: a team that trains and supports other teams in adopting and modifying software as part of a transition or learning period. -- **Complicated subsystem**: a team that is responsible for a subsystem too complicated (mathematics/calculation/...) to be handled by a normal stream-aligned team or platform team. Optional and only used when necessary. - -![](assets/team-toplogies_the-four-fundamental-team-topologies.webp) - -![](assets/team-toplogies_four-fundamental-topologies-shown-with-the-flow-of-change.webp) - -Effective software delivery requires the combination of specific team types, but the interaction modes between these four fundamental team topologies are vitally important to understanding and nurturing effective software delivery: - -- **Collaboration mode**: teams that work together on a shared goal—especially those working to discover new technology or approaches—generally have an advantage over individual workers. The rapid pace of learning is an important asset. -- **X-as-a-Service mode**: One team uses something provided by another team (such as an API, a tool, or a full software product). Collaboration is minimal. -- **Facilitating mode**: one team (usually an enabling team) support another team in learning or adopting a new approach. - -![](assets/team-toplogies_3-team-interaction-modes.webp) - -## References - -- _Team Topologies: Organizing Business and Technology Teams for Fast Flow by Manuel Pais and Matthew Skelton_ -- https://teamtopologies.com/key-concepts diff --git a/topics/engineering/the-10x-engineer.md b/topics/engineering/the-10x-engineer.md deleted file mode 100644 index 275efbd7a..000000000 --- a/topics/engineering/the-10x-engineer.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: The 10x engineer -description: Discover what makes a 10x engineer highly productive with skills in software development, communication, innovation, and problem-solving, and why tech companies seek these exceptional engineers. -date: 2016-03-01 -tags: - - engineering -redirect: - - /8tuFVA ---- - -10x engineer is a concept sometimes used in Silicon Valley to describe an engineer that is 10x more productive than an average engineer although the 10x metric is figurative. Sometimes referred to as "Ninjas", these engineers are highly sought after by all tech companies. - -> Jim: You gave me 100 resumes but none of these guys are 10x engineers. Why hire a few of these guys to slow us down when a 10x engineer is so much more productive? - -10x is a number. Despite the fact that a lot of people calls it a myth, we should know that there are real engineers **who can out perform others**. With that understanding, we do need metrics to keep track the productivity but as a Hiring Manager, **being open minded to look for those candidates is a must**. - -More: - -- [The mythical 10x programmer](http://antirez.com/news/112) -- [10X Engineer](https://gorillalogic.com/blog/10x-engineer-series-part-1-2/) -- [Becoming a 10X engineer](https://hackernoon.com/becoming-a-10x-engineer-is-it-possible-a8cf593392c3) - -## The depict of a 10x engineer - -### Definition - -10x engineers (10xe) must not be mediocre engineers who are in the middle of the pyramid of the ability, productivity and quality, instead they must be above the middle of the pyramid. 10xes are always obsessed with the quality of the output they deliver, outperform the mediocre engineers in terms of specialized (e.g technical/designing) skills, communication, entrepreneurship mindset and skills, and contribution. They are doers, creators, thing breakers, on the way or already innovators to create better things. - -### Mindset - -- Growth Mindset -- Creative -- Innovative -- Contributive - -### Skillset - -Communication - -**Software Engineer** - -- Software Development Life Cycle -- Software Development Process -- Operating System -- Computer Network -- Database -- System Design -- DevOps -- Data Structure and Algorithms -- Programming -- Tooling -- 10x (spirit) -- Tech: Linux; Docker; IDE; Modeling; Programming; Architect; Algo & DS; -- Process: Scrum; Development Process, SDLC -- Communication -- Make thing better -- Entrepreneurship -- Hand-on -- Create / build sth people want: Inventor; Doer -- Contribution: Bring value to people diff --git a/topics/engineering/the-removal-of-apache-kafka-s-dependency-on-zookeeper.md b/topics/engineering/the-removal-of-apache-kafka-s-dependency-on-zookeeper.md deleted file mode 100644 index 0b1e77b52..000000000 --- a/topics/engineering/the-removal-of-apache-kafka-s-dependency-on-zookeeper.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "The removal of Apache Kafka's dependency on Zookeeper" -description: Kafka has been using Zookeeper for a variety of important functions. It uses Zookeeper to keep track of which brokers are part of the Kafka cluster. This is a critical task, as it enables Kafka to ensure that each broker is working properly and that the entire cluster is functioning as intended... -date: 2023-09-13 -authors: - - fuatto -github_id: fuatto -icy: 10 -tags: - - engineering - - kafka - - zookeeper -redirect: - - /Ue22pw ---- - -## Before the removal - -### Importance of Zookeeper - -Kafka has been using [Zookeeper](https://cwiki.apache.org/confluence/display/ZOOKEEPER/ProjectDescription) for a variety of important functions. It uses Zookeeper to keep track of which brokers are part of the Kafka cluster. This is a critical task, as it enables Kafka to ensure that each broker is working properly and that the entire cluster is functioning as intended. - -Additionally, Zookeeper is responsible for electing the leader of a given partition. This is a complex task that requires significant computational power, and Zookeeper is well-suited to handle it. - -Another important function that Zookeeper performs for Kafka is storing configurations for topics/permissions and current offsets of each consumer group. Managing configurations is critical for ensuring that Kafka is able to function properly as a distributed system and that each consumer is receiving the correct information. - -Finally, Zookeeper sends health-check notifications to Kafka for a variety of events. This includes new topics being created, brokers coming up or dying, and topics being deleted. - -### Downsides to Zookeeper - -Despite its many advantages, there are also some downsides to using Zookeeper. One challenge is that the Kafka Controller state or metadata often does not match the Zookeeper state. This can lead to confusion and make it difficult to determine the current state of the cluster. - -Additionally, there can be performance issues when a broker joins or leaves a cluster, as Zookeeper can be overloaded by a high number of leader elections. Despite these challenges, many organizations continue to find Zookeeper to be a valuable tool in managing their Kafka clusters, and Kafka continues to rely on it for its important functions. - -## Why - -The Kafka Controller **state** or **metadata** often does not match the Zookeeper state. Despite this, Kafka uses Zookeeper for several important tasks. For example, it keeps track of which brokers are a part of the Kafka cluster and is responsible for electing the leader of a given partition. Additionally, it stores configurations for topics/permissions and current offsets of each consumer group. Lastly, it sends health-check notifications to Kafka for events such as new topics being created, brokers coming up or dying, and topics being deleted. - -One of the advantages of using Zookeeper is simpler deployment and configuration, as it is a _separate Java service_. However, there are also some downsides to using Zookeeper. Performance issues occur when a broker joins or leaves a cluster, as Zookeeper can be overloaded by a high number of leader elections. In addition, scaling Kafka only supports a limited number of partitions, up to **200k partitions**. - -Despite the potential challenges, Kafka continues to rely on Zookeeper for its important functions, and many organizations continue to find it a valuable tool in managing their Kafka clusters. - -## Architecture - -![](assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka_architecture.webp) - -### In the old architecture - -The controller is a crucial component of the system that manages the state of the Kafka cluster. Upon election, the controller loads its state from a Zookeeper quorum consisting of multiple nodes. This state includes information about topics, their partitions, and the brokers that host these partitions. Once the controller has loaded its state, it begins pushing updates to other nodes in the cluster using `LeaderAndIsr` and `UpdateMetadata` messages. These updates ensure that all nodes in the cluster are aware of the latest state of the system, and that they can take appropriate actions in response to changes in this state. Overall, the controller plays a critical role in ensuring the smooth functioning and reliability of the Kafka cluster. - -![](assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_kafka-controller-apis.webp) - -### In **KRaft Mode** - -Kafka has been using Zookeeper for various critical tasks, such as keeping track of which brokers are part of the Kafka cluster, electing leaders for partitions, storing configurations for topics/permissions, and current offsets of each consumer group, and sending health-check notifications to Kafka. - -Despite its advantages, using Zookeeper also poses challenges when it comes to performance and consistency in Kafka's metadata. For example, Zookeeper can be overloaded by a high number of leader elections, especially when a broker joins or leaves a cluster. Additionally, Kafka Controller state or metadata often does not match the Zookeeper state, leading to confusion and difficulty in determining the current state of the cluster. - -To address these challenges, Kafka introduced KRaft mode, which replaces the Zookeeper quorum with the Controller quorum. The Controller quorum manages all metadata stored in Zookeeper, including topics, partitions, ISRs, and configs, among others. An active controller is elected using [Raft](https://raft.github.io/), a consensus algorithm that does not rely on any external system. The leader controller handles RPC calls from the brokers, while the follower controllers replicate the data written to the active controller to be ready in case the active controller fails. - -With the KRaft mode, the controller quorum does not need to load state from Zookeeper when the leadership changes, which significantly improves performance. Instead, the brokers fetch updates from the active controller via the new `MetadataFetch` API, tracking the offset of the last update fetched and only requesting newer updates. - -Kafka's KRaft mode offers various advantages over Zookeeper, such as simpler deployment and configuration since Zookeeper is a separate Java service. Furthermore, it addresses the performance issues associated with Zookeeper, especially when a broker joins or leaves a cluster, by reducing the number of leader elections. - -In conclusion, KRaft mode is a crucial improvement to Kafka's architecture, ensuring smooth functioning and reliability of the Kafka cluster by providing a better alternative to Zookeeper for managing Kafka's metadata. - -![](assets/the-removal-of-apache-kafka's-dependency-on-zookeeper_time-shutdown-operations-kafka.webp) - -For simple local setup, can refer to [Apache Kafka's quickstart guide](https://kafka.apache.org/quickstart) . For more detailed configurations, you can refer to [Kafka's kraft config](https://kafka.apache.org/documentation/#kraft_config) - -## References - -- [https://www.conduktor.io/kafka/zookeeper-with-kafka/](https://www.conduktor.io/kafka/zookeeper-with-kafka/ "https://www.conduktor.io/kafka/zookeeper-with-kafka/") -- [https://developer.confluent.io/learn/kraft/](https://developer.confluent.io/learn/kraft/ "https://developer.confluent.io/learn/kraft/") -- [https://raft.github.io/](https://raft.github.io/ "https://raft.github.io/") -- [https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum](https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum "https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum") -- [https://kafka.apache.org/documentation/#zk](https://kafka.apache.org/documentation/#zk "https://kafka.apache.org/documentation/#zk") diff --git a/topics/engineering/url-redirect-vs-rewrite.md b/topics/engineering/url-redirect-vs-rewrite.md deleted file mode 100644 index f9728d594..000000000 --- a/topics/engineering/url-redirect-vs-rewrite.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: URL Redirect vs. Rewrite; What’s the difference? -description: "When working with websites and managing the computers that show them, we run into two important ideas: redirect URLs and rewrite URLs. These ideas are like tools that help us make websites work better for people. They help us make websites easier to find on search engines, and they help us make the paths on websites smoother. In this guide, we're going to explore these tools together. We'll learn what makes them different and how we can use them to make websites even cooler." -date: 2023-09-06 -authors: - - trankhacvy -github_id: trankhacvy -icy: 10 -tags: - - engineering - - http - - seo - - url-redirect - - url-rewrite -redirect: - - /dRrkZA ---- - -When working with websites and managing the computers that show them, we run into two important ideas: redirect URLs and rewrite URLs. These ideas are like tools that help us make websites work better for people. They help us make websites easier to find on search engines, and they help us make the paths on websites smoother. In this guide, we're going to explore these tools together. We'll learn what makes them different and how we can use them to make websites even cooler. - -## Redirect URLs: navigating the path - -A redirect URL is a technique used to guide users from one URL to another, often due to a webpage's relocation, restructuring, or a change in content. Imagine it as a virtual signpost that ensures visitors reach their desired destination without wandering in the digital wilderness. Redirects come in various flavors, but the two most common types are: - -### 1. 301 permanent redirect - -The 301 redirect is a magician's wand for SEO. It indicates that a page has moved permanently to a new location, preserving almost all of its link equity and SEO juice. This not only assists users in finding the new location but also guides search engines to update their indexes. - -### 2. 302 temporary redirect - -The 302 redirect is more like a "Come back soon" note. It informs browsers and search engines that a page has temporarily moved. While it does redirect users to a new location, it's not meant for permanent changes. - -**How Redirect URLs Work: A Closer Look** - -1. The user types a URL into the browser's address bar or clicks on a link. -2. The server follows predefined rules, directing requests for the URL to be externally redirected to a new URL. The server then instructs the user's browser to go to this new URL. -3. The browser automatically navigates to the new URL. -4. The server provides the browser with the content of the new URL, which users can then view in their browser. - -## Rewrite URLs: crafting the illusion - -If redirect URLs are signposts, then rewrite URLs are masterful illusions. URL rewriting involves modifying the URL that a user enters or clicks on, usually to make it more user-friendly, descriptive, or structured. This modification is typically done on the server side before the request is processed. The rewritten URL is then used to fetch the appropriate resource or content from the server. - -### 1. clean URLs - -Rewrite URLs can transform complex, dynamic URLs into clean, human-readable versions. For instance, converting `https://example.com/products.php?id=123` into `https://example.com/products/123`. - -### 2. SEO enhancement - -Clean, concise URLs not only appeal to users but also win favor with search engines. Rewriting URLs to include relevant keywords can boost SEO efforts and improve search engine ranking. For instance: - -| Good URL | Bad URL | -| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| https://ahrefs.com/blog/seo-friendly-urls/ | https://ahrefs.com/blog/seo-friendly-urls/?utm_source=google&utm_medium=cpc&utm_campaign=seo | -| https://moz.com/blog/15-seo-best-practices-for-structuring-urls | https://moz.com/blog/15-seo-best-practices-for-structuring-urls?ref=homepage&date=2023-09-05 | -| https://seranking.com/blog/create-seo-friendly-url/ | https://seranking.com/blog/index.php?post_id=1234&create-seo-friendly-url | - -### 3. improved user experience - -Rewrite URLs can enhance the user experience by making URLs more memorable, shareable, and aesthetically pleasing. A user is more likely to click on `https://example.com/contact` than a convoluted URL. - -**How Rewrite URLs Work: A Closer Look** -Now, let's delve into the process of how URL rewriting works using an example: - -1. The user types http://example.com/product/macbook-pro into the browser's address bar or clicks on a link. -2. The server follows predefined rules and rewrites http://my-app/product/macbook-pro to http://my-app/product?id=macbook-pro. The server processes the rewritten URL and returns the content to the browser. - -## Key differences and how to choose - -Redirect URLs and rewrite URLs share a common goal: to lead users to the right place. However, they achieve this goal differently: - -- **Purpose**: Redirect URLs are about physically moving users from one URL to another, while rewrite URLs focus on altering the appearance of URLs. -- **Type of change**: Redirects involve a change in the browser's URL bar, while URL rewriting doesn't change the URL displayed in the browser. - -- **HTTP response**: Redirects typically use HTTP status codes like 301 or 302, signaling browsers and search engines about the nature of the move. URL rewriting doesn't change the HTTP response. - -**Choosing the Right Approach**: If you're moving or restructuring content, use redirect URLs. If you want cleaner, SEO-friendly URLs without changing the content's location, opt for URL rewriting. - -## Conclusion - -Redirect URLs and rewrite URLs are threads that weave together seamless user experiences, enhanced SEO, and organized website structures. By understanding the step-by-step processes of these techniques, you hold the power to guide users through the digital landscape and leave an indelible mark on the virtual world. - -## References - -- https://en.wikipedia.org/wiki/URL_redirection -- https://en.wikipedia.org/wiki/Rewrite_engine -- https://weblogs.asp.net/owscott/rewrite-vs-redirect-what-s-the-difference diff --git a/topics/engineering/what-is-kubernetes.md b/topics/engineering/what-is-kubernetes.md deleted file mode 100644 index 63cca6dc4..000000000 --- a/topics/engineering/what-is-kubernetes.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: What is Kubernetes -description: Kubernetes is an open-source platform for managing containerized applications with features like automated rollouts, self-healing, and load balancing, enabling scalable and portable cloud deployments. -date: 2019-09-27 -tags: - - engineering -redirect: - - /pgLtXQ ---- - -## What is Kubernetes - -Kubernetes is a portable, extensible, open-source platform for managing containerized workloads and services, that facilitates both declarative configuration and automation. It has a large, rapidly growing ecosystem. Kubernetes services, support, and tools are widely available. - -![](assets/what-is-kubernetes_ca7490cb6fe10475c91a359a8177d29c_md5.webp) - -### Traditional deployment era - -Early on, organizations ran applications on physical servers. There was no way to define resource boundaries for applications in a physical server, and this caused resource allocation issues. For example, if multiple applications run on a physical server, there can be instances where one application would take up most of the resources, and as a result, the other applications would underperform. A solution for this would be to run each application on a different physical server. But this did not scale as resources were underutilized, and it was expensive for organizations to maintain many physical servers. - -### Virtualized deployment era - -As a solution, virtualization was introduced. It allows you to run multiple Virtual Machines (VMs) on a single physical server’s CPU. Virtualization allows applications to be isolated between VMs and provides a level of security as the information of one application cannot be freely accessed by another application. - -Virtualization allows better utilization of resources in a physical server and allows better scalability because an application can be added or updated easily, reduces hardware costs, and much more. - -Each VM is a full machine running all the components, including its own operating system, on top of the virtualized hardware. - -### Container deployment era - -Containers are similar to VMs, but they have relaxed isolation properties to share the Operating System (OS) among the applications. Therefore, containers are considered lightweight. Similar to a VM, a container has its own filesystem, CPU, memory, process space, and more. As they are decoupled from the underlying infrastructure, they are portable across clouds and OS distributions. - -Containers are becoming popular because they have many benefits. Some of the container benefits are listed below: - -- Agile application creation and deployment: increased ease and efficiency of container image creation compared to VM image use. -- Continuous development, integration, and deployment: provides for reliable and frequent container image build and deployment with quick and easy rollbacks (due to image immutability). -- Dev and Ops separation of concerns: create application container images at build/release time rather than deployment time, thereby decoupling applications from infrastructure. -- Observability not only surfaces OS-level information and metrics, but also application health and other signals. -- Environmental consistency across development, testing, and production: Runs the same on a laptop as it does in the cloud. -- Cloud and OS distribution portability: Runs on Ubuntu, RHEL, CoreOS, on-prem, Google Kubernetes Engine, and anywhere else. -- Application-centric management: Raises the level of abstraction from running an OS on virtual hardware to running an application on an OS using logical resources. -- Loosely coupled, distributed, elastic, liberated micro-services: applications are broken into smaller, independent pieces and can be deployed and managed dynamically – not a monolithic stack running on one big single-purpose machine. -- Resource isolation: predictable application performance. -- Resource utilization: high efficiency and density. - -## Why we need K8s ? - -- Service discovery and load balancing -- Storage orchestration -- Automated rollouts and rollbacks -- Automatic bin packing -- Self-healing -- Secret and configuration management - -Take full docs at: [https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/) - -## The basic Kubernetes objects - -- [Pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) -- [Service](https://kubernetes.io/docs/concepts/services-networking/service/) -- [Volume](https://kubernetes.io/docs/concepts/storage/volumes/) -- [Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) - -Tutorial page that help you deploy first app: [https://kubernetes.io/docs/tutorials/kubernetes-basics/](https://kubernetes.io/docs/tutorials/kubernetes-basics/) diff --git a/topics/frontend/ThreeJS/assets/cameras-in-threejs_cx1u9zi.webp b/topics/frontend/ThreeJS/assets/cameras-in-threejs_cx1u9zi.webp deleted file mode 100644 index 71d93de53..000000000 Binary files a/topics/frontend/ThreeJS/assets/cameras-in-threejs_cx1u9zi.webp and /dev/null differ diff --git a/topics/frontend/ThreeJS/assets/cameras-in-threejs_oipfgw7.webp b/topics/frontend/ThreeJS/assets/cameras-in-threejs_oipfgw7.webp deleted file mode 100644 index 4ceb4a7ac..000000000 Binary files a/topics/frontend/ThreeJS/assets/cameras-in-threejs_oipfgw7.webp and /dev/null differ diff --git a/topics/frontend/ThreeJS/assets/cameras-in-threejs_uioctax.webp b/topics/frontend/ThreeJS/assets/cameras-in-threejs_uioctax.webp deleted file mode 100644 index 91292a441..000000000 Binary files a/topics/frontend/ThreeJS/assets/cameras-in-threejs_uioctax.webp and /dev/null differ diff --git a/topics/frontend/ThreeJS/assets/cameras-in-threejs_va74b4e.webp b/topics/frontend/ThreeJS/assets/cameras-in-threejs_va74b4e.webp deleted file mode 100644 index 3d1331347..000000000 Binary files a/topics/frontend/ThreeJS/assets/cameras-in-threejs_va74b4e.webp and /dev/null differ diff --git a/topics/frontend/ThreeJS/assets/cameras-in-threejs_zkoiiim.webp b/topics/frontend/ThreeJS/assets/cameras-in-threejs_zkoiiim.webp deleted file mode 100644 index 8a30dc43a..000000000 Binary files a/topics/frontend/ThreeJS/assets/cameras-in-threejs_zkoiiim.webp and /dev/null differ diff --git a/topics/frontend/ThreeJS/cameras-in-threejs.md b/topics/frontend/ThreeJS/cameras-in-threejs.md deleted file mode 100644 index f4b2a8b45..000000000 --- a/topics/frontend/ThreeJS/cameras-in-threejs.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Cameras in ThreeJS -description: "**ThreeJS** is a JavaScript 3D library that allows developers to develop and describe data in 3 dimensions, and then convert them into 2 dimensions and display them on [HTML Canvas](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement). [**Camera**](https://threejs.org/docs/index.html?q=camera#api/en/cameras/Camera) is one of the core elements of a ThreeJS project, beside [Scene](https://threejs.org/docs/index.html?q=scene#api/en/scenes/Scene), [Renderer](https://threejs.org/docs/index.html?q=renderer#api/en/renderers/WebGLRenderer) and 3D objects (can be formed with [Geometries](https://threejs.org/docs/index.html?q=geometry#api/en/geometries/BoxGeometry) and [Materials](https://threejs.org/docs/index.html?q=material#api/en/materials/MeshBasicMaterial))." -date: 2022-09-15 -authors: - - nguyend-nam -github_id: nguyend-nam -tags: - - frontend - - threejs -redirect: - - /pYzwKQ ---- - -**ThreeJS** is a JavaScript 3D library that allows developers to develop and describe data in 3 dimensions, and then convert them into 2 dimensions and display them on [HTML Canvas](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement). [**Camera**](https://threejs.org/docs/index.html?q=camera#api/en/cameras/Camera) is one of the core elements of a ThreeJS project, beside [Scene](https://threejs.org/docs/index.html?q=scene#api/en/scenes/Scene), [Renderer](https://threejs.org/docs/index.html?q=renderer#api/en/renderers/WebGLRenderer) and 3D objects (can be formed with [Geometries](https://threejs.org/docs/index.html?q=geometry#api/en/geometries/BoxGeometry) and [Materials](https://threejs.org/docs/index.html?q=material#api/en/materials/MeshBasicMaterial)). - -This article gives basic information about characteristics of the frequently used types of camera in ThreeJS. - -## Most commonly used types of camera in ThreeJS - -You cannot use the `Camera` class directly to initialize a new camera in ThreeJS. Instead, use the `PerspectiveCamera` or `OrthographicCamera` specifically. The **perspective camera** gives information about depth, distances, etc. since it represents perspective in **real life**. On the other hand, with an **orthographic camera**, all sizes are projected the same way and we can easily compare them accurately. - -The image below shows the object viewed in **perspective camera** (upper) and in **orthographic camera** (lower). - -![](assets/cameras-in-threejs_va74b4e.webp) - -Place them into the **Oxyz** coordinate... - -![](assets/cameras-in-threejs_uioctax.webp) - -### Perspective camera - -![](assets/cameras-in-threejs_oipfgw7.webp) - -Some crucial attributes: - -- **fov** (field of view): Camera frustum **vertical** field of view, i.e. the angle of the view in the yOz plane, in degree. The lower the number, the narrower the view. ![](assets/cameras-in-threejs_zkoiiim.webp) -- **near** and **far**: Clipping planes at 2 points on the z axis. Note that objects with z position outside the `(near, far)` interval will **not** be displayed. - -```javascript -const camera = new THREE.PerspectiveCamera(45, 16 / 9, 1, 1000); -scene.add(camera); -``` - -In the example above, a perspective camera is initialized with `fov` of 45 degree, `aspect-ratio` of 16 / 9, `near` and `far` of 1 and 1000 respectively. - -Perspective camera visualization from [r105.threejsfundamentals.org](https://r105.threejsfundamentals.org/threejs/threejs-cameras-perspective-2-scenes.html) with [`CameraHelper`](https://threejs.org/docs/index.html?q=camera#api/en/helpers/CameraHelper): - - - -### Orthographic camera - -![](assets/cameras-in-threejs_cx1u9zi.webp) - -Some crucial attributes: - -- **left**, **right**, **top** and **bottom**: Horizontal and vertical position of 4 segments of the view. -- **near** and **far**: Same as for perspective camera. - -```javascript -const camera = new THREE.OrthographicCamera(-2, 2, 1, -1, 1, 1000); -scene.add(camera); -``` - -In the example above, an orthographic camera is initialized with `left`, `right`, `top` and `bottom` equal -2, 2, 1 and -1 respectively. The last 2 numbers represent `near` and `far`. - -Orthographic camera visualization from [r105.threejsfundamentals.org](https://r105.threejsfundamentals.org/threejs/threejs-cameras-orthographic-2-scenes.html) with [`CameraHelper`](https://threejs.org/docs/index.html?q=camera#api/en/helpers/CameraHelper): - - - -## Other types of camera - -Beside those 2 most frequently used cameras, ThreeJS also provides various options: - -- [ArrayCamera](https://threejs.org/docs/index.html?q=camera#api/en/cameras/ArrayCamera) -- [CubeCamera](https://threejs.org/docs/index.html?q=camera#api/en/cameras/CubeCamera) -- [StereoCamera](https://threejs.org/docs/index.html?q=camera#api/en/cameras/StereoCamera) - -## Reference - -- https://threejs.org/docs/index.html?q=camera#api/en/cameras/Camera -- https://r105.threejsfundamentals.org/threejs/lessons/threejs-cameras.html -- https://www.vectorstock.com/royalty-free-vector/isometric-and-perspective-drawings-vector-7297379 -- https://wiki.freecadweb.org/index.php?title=File:Orthographic_Perspective.gif -- https://www.celestron.com/blogs/knowledgebase/what-is-the-field-of-view-of-a-pair-of-binoculars diff --git a/topics/frontend/a-quick-intro-to-webassembly.md b/topics/frontend/a-quick-intro-to-webassembly.md deleted file mode 100644 index 7f0b678ae..000000000 --- a/topics/frontend/a-quick-intro-to-webassembly.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: A quick intro to WebAssembly -description: Discover how WebAssembly, a fast, portable binary format designed for running C, C++, and Rust on the web, outperforms JavaScript by enabling near-native execution speeds in browsers and servers. -date: 2020-06-15 -tags: - - wasm -redirect: - - /Vk8kUw ---- - -If you haven’t heard of WebAssembly yet, then you will soon. It’s one of the industry’s best-kept secrets, but it’s everywhere. It’s supported by all the major browsers, and it’s coming to the server-side, too. It’s fast. It’s being used for gaming. It’s an open standard from the World Wide Web Consortium (W3C), the main international standards organization for the web. - -## Format definition - -WebAssembly (abbreviated Wasm) is a **binary instruction format** for a stack-based virtual machine. Wasm is designed as a portable target for compilation of high-level languages like C/C++/Rust, enabling deployment on the web for client and server applications. - -## Why WebAssembly? - -### Recall of assembly - -In the old days, when you had to work on a computer to do something like for example, add two numbers and print the result, you had to write instructions in the binary language. These instructions were specific to an architecture of a processor like the 8086 microprocessor or x86 processor. These binary instructions are collectively called the machine code. However, writing machine code with bare hands was a tedious and error-prone task. Also, reading a machine code was at times impossible. - -Hence, the text-based Assembly language was created. This **Assembly language was human-readable and one could easily program or read the Assembly language**. To convert the Assembly code into machine code, a compiler AKA assembler is used (along with the linker). **Assembly language is the lowest form of abstraction over machine code since it directly compiles to the machine code of the given processor architecture.** - -Other compiled languages like C or C++ also compile to machine code, however, these languages are processor agnostic. This means the compiler used by these languages does the actual job of compilation to machine code. - -However, these languages don’t always guarantee the performance similar to the Assembly language because as the program grows larger and many program files are involved, the compiler has to make predictions on how to best compile the program. - -Hence, the machine code generated from these languages may not be very optimized and may run slower. This also means, no language like C or C++ gets close to generating a very optimized machine code like the Assembly language can do. - -### What WebAssembly actually is - -The term **WebAssembly (AKA Wasm)** is inspired by the Assembly language since it is the lowest form of text-based human-readable language which generates very optimized and blazing fast programs in machine code. - -What WebAssembly is trying to achieve is **to create a language that can run as fast as possible, closer to Assembly speeds but on the Web. Even though WebAssembly is a language in itself, its main intention is to create a toolchain for other programming languages like C, C++, Rust, etc. to compile directly to WebAssembly**. This way, web programmers can write programs in the language of their choice and run inside a browser. But as we know, only JavaScript language can be understood by the web browsers and JavaScript is not so popular when it comes to speed, then how WebAssembly is going to run inside a browser and speed things up? - -### What about Javascript? - -JavaScript is an **interpreted language**. This means we do not have to compile the JavaScript source code before sending it to the browser. An interpreter can take the raw JavaScript code and run it for you. - -JavaScript is also a **dynamically typed language**, unlike C and C++. This means variables declared using var can store any type of data type like int, string, boolean and also complex data types like object and array. **The lack of type system is what makes JavaScript slow to run**. A statically typed language can produce a much efficient machine code because of the information it has about the data like its type and size. - -**So whenever you think, a statically typed language like C or C++ is making your life a living hell for no reason, think about the performance.** - -### Historical reason of poor performance in JavaScript - -You might ask, why JavaScript was designed this way if it is so poor when it comes to speed? For that, we need to understand its history. - -In the initial days of Web, web browsers were used to display static pages. Normally these pages were non-interactive. **To add some interaction**, a new language was introduced in the Netscape browser back in 1995 by Brendan Eich. This new language was **JavaScript (previously called the LiveScript) and it took 10 days for him to design it.** - -Knowing that Java was a rich, complex, compiled language aimed at professional programmers, Netscape and others also wanted a lightweight interpreted language to complement Java. This language would need to appeal to nonprofessional programmers much like Microsoft's Visual Basic and interpretable for easy embedding in webpages - -> If I had done classes in JavaScript back in May 1995, I would have been told that it was too much like Java or that JavaScript was competing with Java … I was under marketing orders to make it look like Java but not make it too big for its britches … [it] needed to be a silly little brother language. - -**Nothing good can come out of 10 days but for 10 days worth of effort, JavaScript was a marvel**. Other languages and plugins like ActionScript, Silverlight, and Flash came along but **JavaScript won the battle.** - -JavaScript was not designed by considering the performance in mind. It had to just work inside a browser and provide API to work with DOM. But since many browsers tried to adopt it in their own way, it had to be standardized. - -**Ecma International is the standards organization that standardizes JavaScript and the Technical Committee 39 (TC39) manages this standard**. This standard is known as EcmaScript and the EcmaScript phrase is also used interchangeably with JavaScript since JavaScript trademark is owned by Oracle Corporation. - -### How does Javascript work? - -**EcmaScript specification tells how JavaScript should be implemented by the browser so that a JavaScript program runs exactly the same in all the browsers, but it does not tell how JavaScript should run inside these browsers. It is up to the browser vendor to decide.** - -Every browser provides a JavaScript engine that runs the JavaScript code. The Netscape browser used the SpiderMonkey JavaScript engine. This engine was a rudimentary interpreter with no optimizations. Running the JavaScript code with this engine was slow but it worked. - -As you can see from the diagram above, the job of the first JavaScript engine was to take the JavaScript source code and compile it to the binary instructions (machine code) that a CPU can understand. - -A rudimentary JavaScript engine contains a baseline compiler whose job is to compile JavaScript source code into an intermediate representation (IR) which is also called the bytecode and feeds this bytecode to the interpreter. The interpreter takes this bytecode and converts to the machine code which is eventually run on the machine’s hardware (CPU). - -This is *just like how Java works but the bytecode generation is done by the programmer and bytecode is shared universally rather than the source code*. - -A baseline compiler’s job is to compile code as fast as possible and generate less-optimized bytecode (or machine code in other cases). **Since the interpreter has an unoptimized bytecode to work with, the application speed will be slow, however, the application bootstrap time will be very less.** - -> _SpiderMoney JavaScript has evolved into a piece of complex machinery to produce highly optimized machine code and currently used in the Firefox browser. You can follow this documentation for the source code._ - -When it comes to a **highly dynamic and interactive web application,** the user experience is very poor with this model of JavaScript execution. This problem was faced by **Google’s Chrome browser while displaying Google Maps on the web**. To increase the JavaScript performance on the web, they had to come up with a better approach. Google Chrome from the early days uses the **V8 JavaScript engine.** In the beginning, to improve the JavaScript performance, they added two pieces in their JavaScript engine pipeline as shown below. - -In the 2010 version of the V8 JavaScript engine, there were two main pieces of machinery that did the heavy lifting for the engine. The full-codegen was the **baseline compiler** whose job was to **spit out unoptimized machine code as fast as possible for faster application bootstrap.** As the application was running, the **crankshaft compiler** would **kick in and optimize the source code and replace the parts of the machine code generated by the baseline compiler**. This optimization would result in better application performance as better and better machine code is generated. *However, this process comes with the cost of large CPU overhead and memory consumption. Hence V8 has to come up with another model.* - -The above version of the JavaScript engine does not contain an interpreter. **This is a JIT (Just-In-Time) compilation model** as code is compiled to the machine level on the fly and later optimized, also to the machine code. - -## How JavaScript is optimized? - -There are various criteria for optimizing JavaScript code. Before JavaScript code is passed to the interpreter or baseline compiler, it has to first get parsed into an Abstract Syntax Tree (AST) which is a tree-like structure of the code. - -_When we run a JavaScript application, we do not need all the code at the application startup time_. For example, if we have a function that is called on the user action, like a button click, that code can be parsed later. - -Identifying things that need to be parsed immediately and generating machine code is the best strategy for faster application bootstrap. Sometimes, JavaScript code contains unnecessary complex logic that can be simplified. For example, a `for` to increment an integer can be inlined using `+` operations n number of times. This process is called Loop unrolling. Similar optimizations can be made using function inlining. - -**The lack of type system in JavaScript is what makes JavaScript engine produce less optimized machine code.** Hence, based on already defined values, a JavaScript engine can guess the data types of the variables and generate better machine code. - -Meanwhile, what JavaScript engine can also do is **gather profiling data of the code execution and look for the code that runs slower.** This code is called **the “Hot” code** perhaps because it burns the CPU. This code can be further optimized and replaced with an optimized machine code. Considering these things in mind and other problems caused by full-codegen and crankshaft, the V8 team created a new version of the V8 engine from the ground up. This new version of the JavaScript engine was released in 2017. - -As you can see from the above figure, the V8 team introduced a new interpreter pipeline Ignition whose job was to generate the bytecode from the JavaScript source code using a baseline compiler and later interpret that bytecode using an interpreter. - -The **TurboFan optimization compiler can optimize this bytecode in the background**(in separate threads) as the application is running and generate a very optimized machine code that will be replaced eventually. Turbofan receives the profiling data from the Ignition interpreter and looks for the code that is Hot. **It can make the guesses on how to optimize the code better (by guessing the data types) and optimize or de-optimize the code**. - -## The invention of asm.js - -So far we have understood that a lot of throughs, efforts and money have been put into developing JavaScript engines to cope up with complex JavaScript applications and somehow, make it faster. - -When everybody was working hard to develop faster JavaScript engines, a team at Mozilla went off the books. Back in 2013, they created a subset of JavaScripti which has the feature of statically typed language and manual memory management. They called it the asm.js. - -```javascript -function add(a, b) { - return a + b; -} -``` - -The add function takes two values and returns the concatenated value (sum). When we want to generate a highly optimized machine code, we need the data type of the variable arguments a and b. However, we don’t have that in JavaScript. - -Even if had to make guess, we can’t be sure. *Because a and b can be integers or strings or a mix of both. But let’s say, we were expecting only 32-bit integers, how possibly we can inform this to a JavaScript engine?* - -This is where asm.js specifications come into the picture - -```javascript -function add(a, b) { - a = a | 0; - b = b | 0; - return (a + b) | 0; -} -``` - -In the modified code above, we are overriding `a` and `b` with the value of `a` and `b` value respectively but with a binary or condition. What this would do is to convert the values of `a and b int 32-bit integers`. - -The asm.js specification specifies *only 3 types* that you can use in your JavaScript code, `the 32-bit integer`, and `the 32-bit` & `64-bit floating-point numbers`. This makes your code easy to compile with high precision. - -However, how this code is compiled and converted to machine code depends on the JavaScript engine behind the scenes. The first support of asm.js came in the SpiderMonkey engine of the Firefox browser. - -If our JavaScript code contains `"use asm";` annotation and the JavaScript code has been written according to asm.js specifications, SpiderMonkey could efficiently convert the JavaScript code into optimized machine code. - -After the successful proof-of-concept demonstration by the Mozilla team, other browser vendors like Chrome and Edge rolled out support for the asm.js specifications. Applications written in asm.js were relatively faster than their counterparts written in normal JavaScript. - -Apart from a virtual type system, asm.js specification tells us to write our JavaScript code inside a function called as a module. We need to **instantiate this module by providing an ArrayBuffer which acts like a heap memory.** **By abstracting the memory of our module from the memory of the main JavaScript thread**, we don’t have the necessary overhead of dynamic memory management and garbage collection provided by the JavaScript engines. Let’s write a sample asm.js module by hand. - -But first, let's understand the asm.js module structure - -```javascript -function MyAsmModule(stdlib, foreign, heap) { - "use asm"; - - // module body... - return { - export1: f1, - export2: f2, - // ... - }; -} -``` - -From the example above, `MyAsmModule` function is the `asm.js` module that we will *instantiate later*. Let’s understand the arguments to this function and return value. - -- The `stdlib` argument is an object that contains standard JavaScript libraries accepted in asm.js specifications (listed here). -- The `foreign` object contains references to the external JavaScript functions that our module depends on, also called the foreign function interface (FFI). -- The `heap` argument is the raw ArrayBuffer which will be used as a heap for memory storage optionally required by the module. - -In the end, our asm.js module has to export some functions which will be consumed by a JavaScript program. Let’s use the add function as one of the exports of our asm.js module and instantiate with a *1kb heap memory.* - -```javascript -function Calc(stdlib, foreign, heap) { - "use asm"; - function add(a, b) { - a = a | 0; - b = b | 0; - return (a + b) | 0; - } - return { - add: add, - }; -} - -var stdlib = null; -var foreign = null; -var heap = new ArrayBuffer(1000); // 1kb - -// create module instance -var calc = Calc(stdlib.foreign, heap); - -// call `add` function -var result = calc.add(1, 2); -console.log(result); -``` - -In the above example, we are telling the JavaScript engine that *we want to run this JavaScript code as asm.js module with the help of* `"use asm";` annotation. We have also provided data types of the function parameters and return value. This works just fine, as you can see the result in the console. - -However, even though you can write asm.js modules by hand, *it is not feasible for large scale projects*. But, *there are some toolchains available to compile programs* written in a statically typed language to asm.js. For example, the add function can be compiled from this C code. - -```javascript -int add( int a, int b ) { - return a + b; -} -``` - -## How asm.js code runs faster - -If a **browser’s JavaScript engine** is capable of understanding asm.js code, then it will **treat asm.js JavaScript differently** than other normal JavaScript code. - -The first thing it will do is **compile JavaScript code to machine code with high precision since the** `asm.js` code contains type information beforehand. The machine code generated from the asm.js code is close to the Assembly machine code. - -Also, it will **keep heap of the asm.js module different from the main JavaScript thread since it does not need garbage collection and tracking**. Since an `asm.js` module manages its memory manually, the overall performance of the code will be better than code that needs dynamic memory allocations and management. Even after with such convincing reasons, \*`asm.js` specifications were never standardized` and it needed a fresher perspective. This is **where WebAssembly comes in and solves the problems of asm.js.** - -_The asm.js specifications are not standardized and it is obsoleted by WebAssembly._ - -### The inception of WebAssembly - -`asm.js` was a successful experiment *but writing optimized JavaScript code by hand was tough and error-prone*. There are *no standard build-toolchains to convert your JavaScript code into* `asm`.js, because it’s not an easy road either. - -On the other hand, `asm.js` code *adds extra complexity in the application* and may at times, *the optimized JavaScript code size is larger than the unoptimized one*, making it harder to transfer over the network. - -Considering these things in mind, `asm.js` needed an overhaul. This optimized code that a JavaScript engine can run should be as light as possible. Also, *it needed to be standardized* so that all the browsers can support it. To address these issues, `the Minimal Viable Product (MVP) development of WebAssembly started as a collaborative effort between Mozilla, Google, and other teams around 2015.` - -In this effort, the first blueprint of WebAssembly was conceptualized. **WebAssembly is a binary format of instructions just like a machine code but for a stack machine.** - -**What is a stack machine?** - -A `stack machine`, unlike a `register machine`, works with a `stack`. A stack is a data structure to store some data in linear order. You can push a value on the stack and it will go on the top of the previous value in the stack. However, you can not pull a value from anywhere. You can only pop or pull the value that is on the top of the stack. diff --git a/topics/frontend/an-introduction-to-atomic-css.md b/topics/frontend/an-introduction-to-atomic-css.md deleted file mode 100644 index 8a30aed3a..000000000 --- a/topics/frontend/an-introduction-to-atomic-css.md +++ /dev/null @@ -1,236 +0,0 @@ ---- -title: An introduction to atomic CSS -description: This article provides an overview of Atomic CSS, including its definition, variations, purposes, and how it compares to other CSS approaches. -date: 2022-10-13 -authors: - - ngolapnguyen -github_id: ngolapnguyen -tags: - - atomic-css - - css - - frontend -redirect: - - /Y6bTjQ ---- - -## Definition - -A definition to Atomic CSS: - -> Atomic CSS is the approach to CSS architecture that favors small, single-purpose classes with names based on visual function. - -Some might also call it Functional CSS, or [CSS utilities](https://designsystem.digital.gov/utilities/). Basically, you can say an Atomic CSS framework is a collection of the CSS like these: - -```css -.m-0 { - margin: 0; -} - -.text-red { - color: red; -} -``` - -We have quite a few utilities-first CSS framework like [Tailwind CSS](https://tailwindcss.com/), [Windi CSS](https://windicss.org/) and [Tachyons](https://tachyons.io/), etc. And there are also some UI libraries that come with some CSS utilities as the complement to the framework, for example [Bootstrap](https://getbootstrap.com/docs/5.1/utilities/api/) and [Chakra UI](https://chakra-ui.com/docs/features/style-props). - -## Variations - -### Static - -Similar to how we write normal CSS, we define a unit-based design system for spacing, color, typography, etc. Take a look at these default utility classes from Tailwind: - -```css -# 1 unit ~ 0.25rem .m-4 { - margin: 1rem; -} - -.m-8 { - margin: 2rem; -} -``` - -**Advantages** - -- Easy to set up and use. -- Immutable styles that speak for themselves (by good naming convention). - -### Programmatic - -This style involves using a build tool to automatically generate styles based on what it finds in the HTML. - -For example, given this: - -```html -
Lorem ipsum
-``` - -The following CSS would be generated: - -```css -.Bgc\(#0280ae\) { - background-color: #0280ae; -} -.C\(#fff\) { - color: #fff; -} -.P\(20px\) { - padding: 20px; -} -``` - -Another example would be Tailwind's JIT mode: - -```html -
...
-``` - -**Advantages** - -- Easy to use. -- Stylesheets generated during the build process are fully optimized with no unused styles. -- Much more flexible & spontaneous (... just like inline style). - -## Purposes - -### Atomic CSS keeps things simple - -- Classes are immutable. They only do one thing. -- Naming is straight forward. It's easy to tell what a class does at a glance. -- Using a utility class in an HTML file is unambiguous - we know what we are changing, without concern of breaking something, somewhere else. -- Easy to pick-up without diving too deep into actual stylesheets. - -### Atomic CSS keeps things consistent - -- Unit-based so styles are consistent & scalable. Let's look into how Tailwind setup their default styles again: - - `m-4 -> margin: 1rem`, `m-8 -> margin: 2rem` -> 1 unit ~ 0.25rem. With this, when we see `m-40`, we can easily deduce that it's `10rem`. -- Designs that follow an atomic system will be easier for developers to implement. - - An offset of 1 or 2 pixels while dragging components around in Figma might be neglectible, but applying those styles to the app will bloat it with "magic numbers" that feel random & irrelevant. - - With designers & developers agreeing on a common styling system, developers still know what they have to use when encountering "magic numbers" in the design. They can see a `padding: 63px` and know the designers actually meant `padding: 64px` (divisible by 4). - -### Atomic CSS is not inline styles - -Inline styles have long been deemed a bad practice, and many people dislike Atomic CSS because they think _it shares the same suffer with inline styles_. - -Here’s an example: - -> What if we want to change everything with a `.black` class to be navy instead? - -An easy solution would be finding all instances of `.black` with our Text Editor and replace them manually. However, it's a _tedious_ deed, and exactly what is frowned upon by the community. People value _reusability_. - -**But we do have reusability with Atomic CSS** (along with many other things). - -#### Reusability - -Reusability of Atomic CSS comes from the way we use them. With libraries such as React, it's easy to define a _reusable_ component, with all the styles bundled inside. Now changing the color of a widely-used button is a one-line operation: - -```jsx -export const CommonButton = () => { - return ( - - ); -}; -``` - -Combining with suitable tools, we can make it even more flexibile: - -```jsx -import { css } from "@linaria/core"; - -const reusableButtonClassName = css` - @apply p-2 rounded bg-white font-base border-secondary; - line-height: 1; - - @screen md { - @apply font-xl; - } -`; - -const reusableActiveButtonClassName = css` - @apply border-primary; -`; - -export const CommonButton = ({ isActive }) => { - return ( - - ); -}; -``` - -As you can see, we are utilizing the strength of both Atomic CSS and [[CSS in JS]]. - -#### Abstraction - -With atomic classes, it is possible to create abstractions that would be impossible with inline styles. - -```html -

- Inline styles suck. -

-

Badly written CSS isn't very different.

-

Utility classes allow for abstraction.

-``` - -The first two examples shown above would require a manual find-and-replace in order to update styling were the design to change. The styles in the third example can be adjusted in a single place within a stylesheet. - -#### Tooling - -Sass, Less, PostCSS, Autoprefixer… The CSS community has created a lot of useful tools that weren’t available for inline styles. - -#### Brevity - -Rather than writing out verbose inline styles, atomic classes can be terse abbreviations of declarations. It’s less typing: `mt-0` vs `margin-top: 0`, `flex` vs `display: flex`, etc. - -#### Specificity - -Inline styles have the 2nd highest level of specificity (only less than `!important`) .The lower specificity of atomic classes compared to inline styles is a good thing. It allows more versatility. - -#### Possibilities - -Utility classes can do things inline styles can't. - -Inline styles do not support media queries, pseudo selectors, `@supports`, or CSS animations. Perhaps you have a single hover effect you want to apply to disparate elements rather than to only one component. - -```css -.circle { - border-radius: 50%; -} - -.hover-radius-0:hover { - border-radius: 0; -} -``` - -Simple reusable media query rules can also be turned into a utility class. Its common to use a classname prefix for small, medium and large screen sizes. Here is an example of a flexbox class that will only apply on medium and large screen sizes: - -```css -@media (min-width: 600px) { - .md-flex { - display: flex; - } -} -``` - -## Atomic CSS is not all or nothing - -There's no doubt CSS utilities can cover a lot of common use-cases, but it's not all we need. There are plenty of cases where utility classes aren’t the best option: - -- If you need to change a lot of styles for a particular component inside of a media query. -- If you want to change multiple styles conditionally during run-time, based on a component's state. - -Refer back to [this section](#Reusability) for a sample scenario. Utility classes can coexist with other approaches. It's just a good idea to define base styles and sane defaults globally. - -## References - -- https://antfu.me/posts/reimagine-atomic-css -- https://css-tricks.com/growing-popularity-atomic-css/ -- https://css-tricks.com/lets-define-exactly-atomic-css/ diff --git a/topics/frontend/applying-mock-service-worker-msw-for-seamless-web-development.md b/topics/frontend/applying-mock-service-worker-msw-for-seamless-web-development.md deleted file mode 100644 index 89cad84b6..000000000 --- a/topics/frontend/applying-mock-service-worker-msw-for-seamless-web-development.md +++ /dev/null @@ -1,170 +0,0 @@ ---- -title: Applying mock service worker (MSW) for seamless web development -description: Mock Service Worker (MSW) is an API mocking library that leverages the Service Worker API to intercept requests. It offers unique features that set it apart from traditional mocking libraries, making it a go-to choice for developers. With MSW, you can seamlessly mock both RESTful and GraphQL APIs, providing flexibility for various API architectures. Additionally, MSW supports both Node.js and browser environments, enabling consistent API mocking across different parts of your application. -date: 2023-06-19 -authors: - - hthai2201 -github_id: hthai2201 -tags: - - api-mocking - - frontend - - mock-service-worker - - web-development-tool -redirect: - - /P9WdMw ---- - -## Introduction - -Mock Service Worker (MSW) is an API mocking library that leverages the Service Worker API to intercept requests. It offers unique features that set it apart from traditional mocking libraries, making it a go-to choice for developers. With MSW, you can seamlessly mock both RESTful and GraphQL APIs, providing flexibility for various API architectures. Additionally, MSW supports both Node.js and browser environments, enabling consistent API mocking across different parts of your application. Let's explore how MSW empowers developers to reliable web applications by seamlessly working in both Node.js and browser environments - -## Applying MSW for API interception in development - -MSW operates client-side by using a Service Worker to intercept requests. However, we don't have to write any of the worker's code by ourselves, but rather copy the worker file distributed by the library. This CLI will help us to do that - -```bash -npx msw init --save -``` - -To configure the worker: - -```js -// src/mocks/browser.js -import { setupWorker } from "msw"; -import { handlers } from "./handlers"; // list of request handlers - -// This configures a Service Worker with the given request handlers. -export const worker = setupWorker(...handlers); -``` - -To start the worker: - -```js -// src/index.js -import React from "react"; -import ReactDOM from "react-dom"; -import App from "./App"; - -if (process.env.NODE_ENV === "development") { - const { worker } = require("./mocks/browser"); - worker.start(); -} - -ReactDOM.render(, document.getElementById("root")); -``` - -Let's consider a scenario where we have an `App` component, as shown in the code snippet below. This component makes an API request, which may not be fully implemented on the backend side, or we might want to test how our component handles in different response data or error responses - -```js -// src/App.js -function App() { - const { data, error } = useFetch("https:/example.com/api/usage"); - if (error) { - return
{error.message}
; - } - return ( -
-

API mocking example

-
{data.firstName}
-
- ); -} -``` - -Now, we can mock api response by adding MSW handlers - -```js -// src/mocks/handlers.js -import { rest } from "msw"; - -export const handlers = [ - //respond with mock data - rest.get("https:/example.com/api/usage", (req, res, ctx) => { - return res( - ctx.status(301), - ctx.json({ - id: 1, - firstName: "Dwarves", - }), - ); - }), - // Alternatively, throw an error - rest.get("https:/example.com/api/usage", (req, res, ctx) => { - return res(ctx.status(500), ctx.json({ message: "Internal Server Error" })); - }), -]; -``` - -This allows us to verify that our component behaves correctly and handles various API responses or errors gracefully, even if the backend is not fully implemented or we want to test different scenarios without relying on the actual API response. - -## Applying MSW for React testing - -One of the most common use cases for MSW is leveraging its request handlers for integration tests. MSW allows you to seamlessly incorporate mocking into any Node process To configure the server: - -```js -// src/mocks/server.js -import { setupServer } from "msw/node"; -import { handlers } from "./handlers"; - -// This configures a request mocking server with the given request handlers. -export const server = setupServer(...handlers); -``` - -Before running your test, start the MSW server: - -```js -beforeAll(() => server.listen()); -``` - -After each test, make sure to clean up by stopping the server and resetting any request handlers that were added on runtime: - -```js -afterEach(() => { - server.resetHandlers(); - cleanup(); -}); - -afterAll(() => server.close()); -``` - -Basic example of an integration test over the `App` component. - -```js -import React from "react"; -import { render, screen } from "@testing-library/react"; -import App from "./App"; - -test("passes", async () => { - render(); - - expect( - // Expect the mocked response to be present in the DOM. - await screen.findByText(`Dwarves`), - ).toBeInTheDocument(); -}); -``` - -you also can add additional handlers during runtime to handle different scenarios in different tests. Here's an example: - -```js -test("handle error responses", async () => { - render(); - // a runtime handler which - server.use( - rest.get("https:/example.com/api/usage", (req, res, ctx) => { - return res(ctx.json({ message: "Internal Server Error" })); - }), - ); - const errorElement = screen.getByText("Internal Server Error", { - className: "error", - }); - expect(errorElement).toBeInTheDocument(); -}); -``` - -The `server.use` function allows us to add a runtime handler to our mock server. However, it's important to note that any handlers added in runtime will be removed when `server.resetHandlers()` is called. - -## Reference - -- [Mocking API servers with mock service worker (MSW)](https://blog.openreplay.com/mocking-api-servers-with-mock-service-worker-msw/) -- [MSW documentation](https://mswjs.io/docs/) diff --git a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215600.webp b/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215600.webp deleted file mode 100644 index 22afbae96..000000000 Binary files a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215600.webp and /dev/null differ diff --git a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215643.webp b/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215643.webp deleted file mode 100644 index 69d361e89..000000000 Binary files a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215643.webp and /dev/null differ diff --git a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215646.webp b/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215646.webp deleted file mode 100644 index 6e3aa101b..000000000 Binary files a/topics/frontend/assets/202210162154---the-best-of-css-tldr_pasted-image-20221016215646.webp and /dev/null differ diff --git a/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_149cb7501d21ad52e476f168b93085cc_md5.webp b/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_149cb7501d21ad52e476f168b93085cc_md5.webp deleted file mode 100644 index 90ec6e4c7..000000000 Binary files a/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_149cb7501d21ad52e476f168b93085cc_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp b/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp deleted file mode 100644 index 236abafe2..000000000 Binary files a/topics/frontend/assets/accelerate-project-initiation-with-advanced-nextjs-boilerplate-react-toolkit_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/adoption-of-pnpm_4601b8f71eebe7c7fbc4a6fc7925a8b6_md5.webp b/topics/frontend/assets/adoption-of-pnpm_4601b8f71eebe7c7fbc4a6fc7925a8b6_md5.webp deleted file mode 100644 index 50f3ae758..000000000 Binary files a/topics/frontend/assets/adoption-of-pnpm_4601b8f71eebe7c7fbc4a6fc7925a8b6_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_atom-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_atom-atomic-design.webp deleted file mode 100644 index 5cadc787a..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_atom-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_atom-design-structure.webp b/topics/frontend/assets/atomic-design-pattern_atom-design-structure.webp deleted file mode 100644 index 2c762593a..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_atom-design-structure.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_folder-structure-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_folder-structure-atomic-design.webp deleted file mode 100644 index d2c4939b0..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_folder-structure-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_molecule-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_molecule-atomic-design.webp deleted file mode 100644 index 569faed53..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_molecule-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_organisms-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_organisms-atomic-design.webp deleted file mode 100644 index 15021cc65..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_organisms-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_page-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_page-atomic-design.webp deleted file mode 100644 index 495a82f04..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_page-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/atomic-design-pattern_template-atomic-design.webp b/topics/frontend/assets/atomic-design-pattern_template-atomic-design.webp deleted file mode 100644 index cb25afcd4..000000000 Binary files a/topics/frontend/assets/atomic-design-pattern_template-atomic-design.webp and /dev/null differ diff --git a/topics/frontend/assets/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue_fca70866d812e626a871c73732276b8b_md5.webp b/topics/frontend/assets/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue_fca70866d812e626a871c73732276b8b_md5.webp deleted file mode 100644 index 76427ec71..000000000 Binary files a/topics/frontend/assets/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue_fca70866d812e626a871c73732276b8b_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/concurrency-in-javascript_javascript-concurrency-model.webp b/topics/frontend/assets/concurrency-in-javascript_javascript-concurrency-model.webp deleted file mode 100644 index 0666f7d6c..000000000 Binary files a/topics/frontend/assets/concurrency-in-javascript_javascript-concurrency-model.webp and /dev/null differ diff --git a/topics/frontend/assets/continuous-translation_0ca3440b9dbd840d349d587dd7fd6a1d_md5.webp b/topics/frontend/assets/continuous-translation_0ca3440b9dbd840d349d587dd7fd6a1d_md5.webp deleted file mode 100644 index 7abd07a54..000000000 Binary files a/topics/frontend/assets/continuous-translation_0ca3440b9dbd840d349d587dd7fd6a1d_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/continuous-translation_5d971ebb8af780ed2ad9e7626daf0d8c_md5.webp b/topics/frontend/assets/continuous-translation_5d971ebb8af780ed2ad9e7626daf0d8c_md5.webp deleted file mode 100644 index 9e609261c..000000000 Binary files a/topics/frontend/assets/continuous-translation_5d971ebb8af780ed2ad9e7626daf0d8c_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/continuous-translation_eaf982ca480d0677ec9d8fd34553b51a_md5.webp b/topics/frontend/assets/continuous-translation_eaf982ca480d0677ec9d8fd34553b51a_md5.webp deleted file mode 100644 index d6b9daa6e..000000000 Binary files a/topics/frontend/assets/continuous-translation_eaf982ca480d0677ec9d8fd34553b51a_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/debugging-in-javascript_chrome_devtools's_console_tab.webp b/topics/frontend/assets/debugging-in-javascript_chrome_devtools's_console_tab.webp deleted file mode 100644 index dc17bd554..000000000 Binary files a/topics/frontend/assets/debugging-in-javascript_chrome_devtools's_console_tab.webp and /dev/null differ diff --git a/topics/frontend/assets/debugging-in-javascript_debugger_example.webp b/topics/frontend/assets/debugging-in-javascript_debugger_example.webp deleted file mode 100644 index ee3102b5f..000000000 Binary files a/topics/frontend/assets/debugging-in-javascript_debugger_example.webp and /dev/null differ diff --git a/topics/frontend/assets/deno_gygtlws.webp b/topics/frontend/assets/deno_gygtlws.webp deleted file mode 100644 index 9f116648d..000000000 Binary files a/topics/frontend/assets/deno_gygtlws.webp and /dev/null differ diff --git a/topics/frontend/assets/deno_kfo8ecl.webp b/topics/frontend/assets/deno_kfo8ecl.webp deleted file mode 100644 index e296ffa48..000000000 Binary files a/topics/frontend/assets/deno_kfo8ecl.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_compositing.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_compositing.webp deleted file mode 100644 index 83f158214..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_compositing.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_cssom.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_cssom.webp deleted file mode 100644 index 99439250e..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_cssom.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_html-parser.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_html-parser.webp deleted file mode 100644 index 1e67fb108..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_html-parser.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layer-tree.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layer-tree.webp deleted file mode 100644 index 24e9ee716..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layer-tree.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layout-tree.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layout-tree.webp deleted file mode 100644 index fffb21fde..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layout-tree.webp and /dev/null differ diff --git a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_render-tree.webp b/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_render-tree.webp deleted file mode 100644 index 70dfd40ba..000000000 Binary files a/topics/frontend/assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_render-tree.webp and /dev/null differ diff --git a/topics/frontend/assets/from-multi-repo-to-monorepo-a-case-study-with-nghenhan-turbo-monorepo_8dc9116f98bf7a170ec249c0e63ad699_md5.webp b/topics/frontend/assets/from-multi-repo-to-monorepo-a-case-study-with-nghenhan-turbo-monorepo_8dc9116f98bf7a170ec249c0e63ad699_md5.webp deleted file mode 100644 index ee0aeb155..000000000 Binary files a/topics/frontend/assets/from-multi-repo-to-monorepo-a-case-study-with-nghenhan-turbo-monorepo_8dc9116f98bf7a170ec249c0e63ad699_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_706c9f06c1ddbab02d7e04195d24bc16_md5.webp b/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_706c9f06c1ddbab02d7e04195d24bc16_md5.webp deleted file mode 100644 index 797e96861..000000000 Binary files a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_706c9f06c1ddbab02d7e04195d24bc16_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_7578257d37750f65869976d63ebb2b57_md5.webp b/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_7578257d37750f65869976d63ebb2b57_md5.webp deleted file mode 100644 index e0e7e66d0..000000000 Binary files a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_7578257d37750f65869976d63ebb2b57_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_8fbb0902507f83afa2b0ef1bc5f830a0_md5.webp b/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_8fbb0902507f83afa2b0ef1bc5f830a0_md5.webp deleted file mode 100644 index c923c2e7b..000000000 Binary files a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_8fbb0902507f83afa2b0ef1bc5f830a0_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_dabaf075b757602a5af2c6bfcead3283_md5.webp b/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_dabaf075b757602a5af2c6bfcead3283_md5.webp deleted file mode 100644 index ab643d05e..000000000 Binary files a/topics/frontend/assets/fundamental-end-to-end-frontend-testing-with-cypress_dabaf075b757602a5af2c6bfcead3283_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/hsl-color_hls-color-pallete.webp b/topics/frontend/assets/hsl-color_hls-color-pallete.webp deleted file mode 100644 index da2459cc7..000000000 Binary files a/topics/frontend/assets/hsl-color_hls-color-pallete.webp and /dev/null differ diff --git a/topics/frontend/assets/hsl-color_hls-color.webp b/topics/frontend/assets/hsl-color_hls-color.webp deleted file mode 100644 index 5ea873ae3..000000000 Binary files a/topics/frontend/assets/hsl-color_hls-color.webp and /dev/null differ diff --git a/topics/frontend/assets/hsl-color_hls-hue.webp b/topics/frontend/assets/hsl-color_hls-hue.webp deleted file mode 100644 index d5fd4b0d6..000000000 Binary files a/topics/frontend/assets/hsl-color_hls-hue.webp and /dev/null differ diff --git a/topics/frontend/assets/hsl-color_hls-lightest.webp b/topics/frontend/assets/hsl-color_hls-lightest.webp deleted file mode 100644 index c33d1a5a2..000000000 Binary files a/topics/frontend/assets/hsl-color_hls-lightest.webp and /dev/null differ diff --git a/topics/frontend/assets/hsl-color_using-hls.webp b/topics/frontend/assets/hsl-color_using-hls.webp deleted file mode 100644 index 35bf5d0f4..000000000 Binary files a/topics/frontend/assets/hsl-color_using-hls.webp and /dev/null differ diff --git a/topics/frontend/assets/html-inert_pasted-image-20220612104313.webp b/topics/frontend/assets/html-inert_pasted-image-20220612104313.webp deleted file mode 100644 index c111ec6cc..000000000 Binary files a/topics/frontend/assets/html-inert_pasted-image-20220612104313.webp and /dev/null differ diff --git a/topics/frontend/assets/i18n-frontend-guideline_0d5e0a2c5a795b96f65caa5b7a360578_md5.webp b/topics/frontend/assets/i18n-frontend-guideline_0d5e0a2c5a795b96f65caa5b7a360578_md5.webp deleted file mode 100644 index ade9ebc9c..000000000 Binary files a/topics/frontend/assets/i18n-frontend-guideline_0d5e0a2c5a795b96f65caa5b7a360578_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/i18n-frontend-guideline_9354a1ef08eeec42a93ec4329cf358c4_md5.webp b/topics/frontend/assets/i18n-frontend-guideline_9354a1ef08eeec42a93ec4329cf358c4_md5.webp deleted file mode 100644 index df1d6030d..000000000 Binary files a/topics/frontend/assets/i18n-frontend-guideline_9354a1ef08eeec42a93ec4329cf358c4_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/mixpanel_dashboard.webp b/topics/frontend/assets/mixpanel_dashboard.webp deleted file mode 100644 index eb0a9bb52..000000000 Binary files a/topics/frontend/assets/mixpanel_dashboard.webp and /dev/null differ diff --git a/topics/frontend/assets/mixpanel_user_activites.webp b/topics/frontend/assets/mixpanel_user_activites.webp deleted file mode 100644 index eff0f97cf..000000000 Binary files a/topics/frontend/assets/mixpanel_user_activites.webp and /dev/null differ diff --git a/topics/frontend/assets/mixpanel_user_profiles.webp b/topics/frontend/assets/mixpanel_user_profiles.webp deleted file mode 100644 index 2cf175b59..000000000 Binary files a/topics/frontend/assets/mixpanel_user_profiles.webp and /dev/null differ diff --git a/topics/frontend/assets/prevent-layout-thrashing_dont-touch-me.webp b/topics/frontend/assets/prevent-layout-thrashing_dont-touch-me.webp deleted file mode 100644 index 200d3f521..000000000 Binary files a/topics/frontend/assets/prevent-layout-thrashing_dont-touch-me.webp and /dev/null differ diff --git a/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing-debug.webp b/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing-debug.webp deleted file mode 100644 index 5f8aa4647..000000000 Binary files a/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing-debug.webp and /dev/null differ diff --git a/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing.webp b/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing.webp deleted file mode 100644 index 5cf7a6655..000000000 Binary files a/topics/frontend/assets/prevent-layout-thrashing_layout-thrashing.webp and /dev/null differ diff --git a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_024be7746f1d8a3f25ffad7888a47caf_md5.webp b/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_024be7746f1d8a3f25ffad7888a47caf_md5.webp deleted file mode 100644 index cae2bc0c5..000000000 Binary files a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_024be7746f1d8a3f25ffad7888a47caf_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_08f14b555ce54599844167b5700622ca_md5.webp b/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_08f14b555ce54599844167b5700622ca_md5.webp deleted file mode 100644 index 4580d6751..000000000 Binary files a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_08f14b555ce54599844167b5700622ca_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_c733b9fb01a2eb50f4a8895d2cd68acd_md5.webp b/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_c733b9fb01a2eb50f4a8895d2cd68acd_md5.webp deleted file mode 100644 index 37eb6d12a..000000000 Binary files a/topics/frontend/assets/recursively-export-file-pattern-in-javascript-es6-application_c733b9fb01a2eb50f4a8895d2cd68acd_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remix-versus-nextjs_a7d49c4d71418c892f72a5c2a7dbee19_md5.webp b/topics/frontend/assets/remix-versus-nextjs_a7d49c4d71418c892f72a5c2a7dbee19_md5.webp deleted file mode 100644 index 384d896b7..000000000 Binary files a/topics/frontend/assets/remix-versus-nextjs_a7d49c4d71418c892f72a5c2a7dbee19_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_04ee21f4b60188ce76c21df96695e6cf_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_04ee21f4b60188ce76c21df96695e6cf_md5.webp deleted file mode 100644 index 743f0e5cc..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_04ee21f4b60188ce76c21df96695e6cf_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_429bb8c82f4e38465293c6f79d782394_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_429bb8c82f4e38465293c6f79d782394_md5.webp deleted file mode 100644 index fd2e63952..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_429bb8c82f4e38465293c6f79d782394_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_50067f1125ee42d2d68068bd93443235_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_50067f1125ee42d2d68068bd93443235_md5.webp deleted file mode 100644 index e6f82f686..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_50067f1125ee42d2d68068bd93443235_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_97c7bfa1adf49d7c225f00976d99eb8c_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_97c7bfa1adf49d7c225f00976d99eb8c_md5.webp deleted file mode 100644 index f5ecef460..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_97c7bfa1adf49d7c225f00976d99eb8c_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_a903c03602f38038a2c4e4eb4344e0b9_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_a903c03602f38038a2c4e4eb4344e0b9_md5.webp deleted file mode 100644 index f32fd28c6..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_a903c03602f38038a2c4e4eb4344e0b9_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_dd640e027b4421ee74d88c2864f0c9ed_md5.webp b/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_dd640e027b4421ee74d88c2864f0c9ed_md5.webp deleted file mode 100644 index 95d0de94c..000000000 Binary files a/topics/frontend/assets/remove-unused-css-styles-from-bootstrap-using-purgecss_dd640e027b4421ee74d88c2864f0c9ed_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/render-optimization-in-data-fetching-libraries_render-optimization-in-data-fetching-1.webp b/topics/frontend/assets/render-optimization-in-data-fetching-libraries_render-optimization-in-data-fetching-1.webp deleted file mode 100644 index 0323602fc..000000000 Binary files a/topics/frontend/assets/render-optimization-in-data-fetching-libraries_render-optimization-in-data-fetching-1.webp and /dev/null differ diff --git a/topics/frontend/assets/retain-scroll-position-in-infinite-scroll_infinite_scrolling.webp b/topics/frontend/assets/retain-scroll-position-in-infinite-scroll_infinite_scrolling.webp deleted file mode 100644 index 335ced146..000000000 Binary files a/topics/frontend/assets/retain-scroll-position-in-infinite-scroll_infinite_scrolling.webp and /dev/null differ diff --git a/topics/frontend/assets/shadow-dom.svg b/topics/frontend/assets/shadow-dom.svg deleted file mode 100644 index 780cd22f6..000000000 --- a/topics/frontend/assets/shadow-dom.svg +++ /dev/null @@ -1 +0,0 @@ -shadowrootdocumentshadowhostDocument TreeShadow BoundaryShadow TreedocumentshadowhostFlattened Tree (for rendering)ShadowTree \ No newline at end of file diff --git a/topics/frontend/assets/singleton-design-pattern-in-javascript_singleton-pattern.webp b/topics/frontend/assets/singleton-design-pattern-in-javascript_singleton-pattern.webp deleted file mode 100644 index 12b925787..000000000 Binary files a/topics/frontend/assets/singleton-design-pattern-in-javascript_singleton-pattern.webp and /dev/null differ diff --git a/topics/frontend/assets/the-fundamental-of-web-performance-pixel-pipeline.webp b/topics/frontend/assets/the-fundamental-of-web-performance-pixel-pipeline.webp deleted file mode 100644 index f1d68749f..000000000 Binary files a/topics/frontend/assets/the-fundamental-of-web-performance-pixel-pipeline.webp and /dev/null differ diff --git a/topics/frontend/assets/the-fundamental-of-web-performance-render-tree.webp b/topics/frontend/assets/the-fundamental-of-web-performance-render-tree.webp deleted file mode 100644 index 115ad2676..000000000 Binary files a/topics/frontend/assets/the-fundamental-of-web-performance-render-tree.webp and /dev/null differ diff --git a/topics/frontend/assets/typesafe-client-server.webp b/topics/frontend/assets/typesafe-client-server.webp deleted file mode 100644 index 1d781f911..000000000 Binary files a/topics/frontend/assets/typesafe-client-server.webp and /dev/null differ diff --git a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_5gtifrk.webp b/topics/frontend/assets/url-formats-for-sharing-via-social-networks_5gtifrk.webp deleted file mode 100644 index d8f131ae9..000000000 Binary files a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_5gtifrk.webp and /dev/null differ diff --git a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_hmijfth.webp b/topics/frontend/assets/url-formats-for-sharing-via-social-networks_hmijfth.webp deleted file mode 100644 index a4d5ae8bd..000000000 Binary files a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_hmijfth.webp and /dev/null differ diff --git a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_yrq15em.webp b/topics/frontend/assets/url-formats-for-sharing-via-social-networks_yrq15em.webp deleted file mode 100644 index 422370155..000000000 Binary files a/topics/frontend/assets/url-formats-for-sharing-via-social-networks_yrq15em.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_01.webp b/topics/frontend/assets/variable-fonts_variable_fonts_01.webp deleted file mode 100644 index 6c35420b3..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_01.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_02.webp b/topics/frontend/assets/variable-fonts_variable_fonts_02.webp deleted file mode 100644 index cd43c0c5a..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_02.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_03.webp b/topics/frontend/assets/variable-fonts_variable_fonts_03.webp deleted file mode 100644 index 4b9b9e8ae..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_03.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_04.webp b/topics/frontend/assets/variable-fonts_variable_fonts_04.webp deleted file mode 100644 index c0708580b..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_04.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_05.webp b/topics/frontend/assets/variable-fonts_variable_fonts_05.webp deleted file mode 100644 index e37f6a994..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_05.webp and /dev/null differ diff --git a/topics/frontend/assets/variable-fonts_variable_fonts_06.webp b/topics/frontend/assets/variable-fonts_variable_fonts_06.webp deleted file mode 100644 index 8efc82a6c..000000000 Binary files a/topics/frontend/assets/variable-fonts_variable_fonts_06.webp and /dev/null differ diff --git a/topics/frontend/assets/wai-aria_r1s3rbm.webp b/topics/frontend/assets/wai-aria_r1s3rbm.webp deleted file mode 100644 index 47e3cfd01..000000000 Binary files a/topics/frontend/assets/wai-aria_r1s3rbm.webp and /dev/null differ diff --git a/topics/frontend/assets/webassembly_wasm-architecture.webp b/topics/frontend/assets/webassembly_wasm-architecture.webp deleted file mode 100644 index d51e50a46..000000000 Binary files a/topics/frontend/assets/webassembly_wasm-architecture.webp and /dev/null differ diff --git a/topics/frontend/assets/webassembly_wasm-how-it-works.webp b/topics/frontend/assets/webassembly_wasm-how-it-works.webp deleted file mode 100644 index 632fb7a74..000000000 Binary files a/topics/frontend/assets/webassembly_wasm-how-it-works.webp and /dev/null differ diff --git a/topics/frontend/assets/webassembly_wasm-js-how-it-work.webp b/topics/frontend/assets/webassembly_wasm-js-how-it-work.webp deleted file mode 100644 index 5fd202791..000000000 Binary files a/topics/frontend/assets/webassembly_wasm-js-how-it-work.webp and /dev/null differ diff --git a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_0f651d536ab6e1811cdf37eb2b15550d_md5.webp b/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_0f651d536ab6e1811cdf37eb2b15550d_md5.webp deleted file mode 100644 index 308d7d7ab..000000000 Binary files a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_0f651d536ab6e1811cdf37eb2b15550d_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_4cde4958507a5ac4d8e7d614175b57de_md5.webp b/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_4cde4958507a5ac4d8e7d614175b57de_md5.webp deleted file mode 100644 index 882c9bb50..000000000 Binary files a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_4cde4958507a5ac4d8e7d614175b57de_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_55e4a8514dc89f283ed5e6b77d839d42_md5.webp b/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_55e4a8514dc89f283ed5e6b77d839d42_md5.webp deleted file mode 100644 index c2804b601..000000000 Binary files a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_55e4a8514dc89f283ed5e6b77d839d42_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_949760adee1b7a897e0b53044b7b0a89_md5.webp b/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_949760adee1b7a897e0b53044b7b0a89_md5.webp deleted file mode 100644 index 4e418169f..000000000 Binary files a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_949760adee1b7a897e0b53044b7b0a89_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_acaaed15e34c391a1ff6b81bbbf6163f_md5.webp b/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_acaaed15e34c391a1ff6b81bbbf6163f_md5.webp deleted file mode 100644 index 9a2d2e0fa..000000000 Binary files a/topics/frontend/assets/what-is-pnpm-compare-to-npmyarn_acaaed15e34c391a1ff6b81bbbf6163f_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-micro-frontend_149cb7501d21ad52e476f168b93085cc_md5.webp b/topics/frontend/assets/why-micro-frontend_149cb7501d21ad52e476f168b93085cc_md5.webp deleted file mode 100644 index 90ec6e4c7..000000000 Binary files a/topics/frontend/assets/why-micro-frontend_149cb7501d21ad52e476f168b93085cc_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-micro-frontend_336c82e3b7bd0e20a3196bd23f043f6a_md5.webp b/topics/frontend/assets/why-micro-frontend_336c82e3b7bd0e20a3196bd23f043f6a_md5.webp deleted file mode 100644 index dda9a3e29..000000000 Binary files a/topics/frontend/assets/why-micro-frontend_336c82e3b7bd0e20a3196bd23f043f6a_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-micro-frontend_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp b/topics/frontend/assets/why-micro-frontend_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp deleted file mode 100644 index 236abafe2..000000000 Binary files a/topics/frontend/assets/why-micro-frontend_8b4ce5b2e752b7bbc96be21b6d2f1349_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-micro-frontend_a5bf635d4856ea99f487589001781c71_md5.webp b/topics/frontend/assets/why-micro-frontend_a5bf635d4856ea99f487589001781c71_md5.webp deleted file mode 100644 index bf2259b74..000000000 Binary files a/topics/frontend/assets/why-micro-frontend_a5bf635d4856ea99f487589001781c71_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-micro-frontend_f956742770614138c3736e182be7da7a_md5.webp b/topics/frontend/assets/why-micro-frontend_f956742770614138c3736e182be7da7a_md5.webp deleted file mode 100644 index 37e242c29..000000000 Binary files a/topics/frontend/assets/why-micro-frontend_f956742770614138c3736e182be7da7a_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_1ff4c200530392d7cfd5ff04c8edb60a_md5.webp b/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_1ff4c200530392d7cfd5ff04c8edb60a_md5.webp deleted file mode 100644 index 559547548..000000000 Binary files a/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_1ff4c200530392d7cfd5ff04c8edb60a_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_974445b14fdc44d726716ff2a1c499a0_md5.webp b/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_974445b14fdc44d726716ff2a1c499a0_md5.webp deleted file mode 100644 index 72c7cd250..000000000 Binary files a/topics/frontend/assets/why-we-chose-our-tech-stack-accelerating-development-with-a-robust-frontend-solution_974445b14fdc44d726716ff2a1c499a0_md5.webp and /dev/null differ diff --git a/topics/frontend/assets/window-and-iframe-communication_iframe-window-content-builder.webp b/topics/frontend/assets/window-and-iframe-communication_iframe-window-content-builder.webp deleted file mode 100644 index a1dc8f9f9..000000000 Binary files a/topics/frontend/assets/window-and-iframe-communication_iframe-window-content-builder.webp and /dev/null differ diff --git a/topics/frontend/assets/window-and-iframe-communication_window-iframe-chatbox.webp b/topics/frontend/assets/window-and-iframe-communication_window-iframe-chatbox.webp deleted file mode 100644 index b5cc896d3..000000000 Binary files a/topics/frontend/assets/window-and-iframe-communication_window-iframe-chatbox.webp and /dev/null differ diff --git a/topics/frontend/atomic-design-pattern.md b/topics/frontend/atomic-design-pattern.md deleted file mode 100644 index 747965a16..000000000 --- a/topics/frontend/atomic-design-pattern.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -title: Atomic design pattern -description: Atomic design is a methodology for crafting design systems with five key fundamental building blocks, which, when combined, promote consistency, modularity, and scalability. -date: 2022-06-15 -authors: - - namtrhg -github_id: namtrhg -tags: - - atomic-design - - frontend -redirect: - - /DhK8NA ---- - -## Atomic design pattern - -Atomic design is a methodology for crafting design systems with five key fundamental building blocks, which, when combined, promote consistency, modularity, and scalability. - -### Why use atomic design - -- Atomic design provides frontend engineers with flexible and consistent designs from the beginning. -- We can move from the abstract to the concrete thanks to Atomic design. As a result, we can design systems that encourage consistency and scalability while also displaying information in its final context. Additionally, by putting a system together rather than taking it apart, we avoid picking out the best patterns later. -- The method fits incredibly well with component-based architectures like React, Vue,... - -## How does it work? - -There are 5 distinct levels of atomic design which are: atoms, molecules, organisms, templates, and pages. - -![](assets/atomic-design-pattern_atom-design-structure.webp) - -### Atoms - -- Atoms are the basic building blocks of matter. When applied to web interfaces, they are our `HTML` tags like an `input` or a `button`. -- These are fairly abstract elements which are often not useful on their own but are good references in the context of a pattern if seen on a bigger picture. - -![](assets/atomic-design-pattern_atom-atomic-design.webp) - -**Molecules** - -Just like in chemistry, molecules are a set of atoms combined. This could include a form input with a title and input standing next to each other or a header containing multiple atoms like icons, buttons, or inputs. - -![](assets/atomic-design-pattern_molecule-atomic-design.webp) - -### Organisms - -Molecules give us some functional building blocks to work with, and they are a sub-set of organisms. Molecules when joined together create organisms which is a relatively complex and distinct section of an interface. - -![](assets/atomic-design-pattern_organisms-atomic-design.webp) - -### Templates - -Templates consist mostly of groups of organisms to form pages, keep in mind that these template doesn't contain data and only contain props. - -Templates begin their life as HTML wireframes, but over time become the final deliverable. - -t this stage we can really see the design coming together and making more sense for the clients to see the layout of the interface before finalizing it. - -![](assets/atomic-design-pattern_template-atomic-design.webp) - -### Pages - -Pages are specific instances of templates. Here, placeholder content is replaced with real representative content to give an accurate depiction of what a user will ultimately see. - -![](assets/atomic-design-pattern_page-atomic-design.webp) - -### Atomic design folder structure - -This is an example of an application folder structure using atomic design, as you can see there are 5 distinct layers and you can organize them in anyway you want following your own preference. - -![](assets/atomic-design-pattern_folder-structure-atomic-design.webp) - -## Reference - -- https://bradfrost.com/blog/post/atomic-web-design/#atoms -- https://andela.com/insights/structuring-your-react-application-atomic-design-principles/ -- https://blog.ippon.tech/atomic-design-in-practice/ diff --git a/topics/frontend/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue.md b/topics/frontend/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue.md deleted file mode 100644 index 3f1acfd9b..000000000 --- a/topics/frontend/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: Choosing the right Javascript framework a deep dive into React vs Angular vs Vue -description: Compare React, Angular, and Vue to find the best JavaScript framework for your web projects by exploring their performance, learning curve, tooling, and industry use cases. -date: 2023-04-24 -tags: - - framework - - frontend - - web -redirect: - - /Do6kuw ---- - -Selecting the ideal JavaScript framework can be a daunting task, particularly when faced with an abundance of highly resourceful and capable options. To make an informed decision that caters to specific project requirements, it's crucial to possess a deep understanding of various critical aspects. In this article, we'll shed light on some of these essential points by conducting a comprehensive comparison of Angular, React, and Vue. - -We’ve had our fair share of decision fatigue keeping up with new frameworks and choosing the right ones for the job. Our aim is to provide you with the necessary insights to determine the most suitable framework for your unique set of projects, helping you navigate the world of JavaScript frameworks with confidence. - -## A brief history of modern frontend development - -Embarking on a journey through the history of frontend development, we first encounter jQuery, a groundbreaking JavaScript library released in 2006 that simplified the process of constructing interactive web applications. As jQuery gained traction, so did the concept of Single-Page Applications (SPAs), which enabled web apps to dynamically rewrite pages without the need for loading new pages entirely. This transformation in web development can be attributed to the ongoing refinement and structuring of front-end frameworks. - -In 2010, AngularJS emerged as a pioneer in the world of frontend development, introducing revolutionary concepts such as two-way data binding, dependency injection, and modular architecture. These features facilitated the creation of maintainable, testable, and scalable applications for developers. - -React, a brainchild of Facebook, was unveiled in 2013 and rapidly gained favor for its effective approach to building user interfaces with reusable components. Pioneering a new way to develop SPAs using the Virtual DOM, React enabled enhanced performance and more efficient updates. The framework's component-based architecture, unidirectional data flow, and thriving ecosystem made it an instant hit among developers. - -Angular, first launched in 2016 as a comprehensive rewrite of AngularJS, assimilated the lessons learned from its precursor and further refined them. Providing a stable, built-in solution without sacrificing size or speed, Angular quickly carved out a niche in the development community. - -In 2014, [Evan You](https://twitter.com/youyuxi), who previously worked at Google and built applications with AngularJS, unveiled Vue, a progressive frontend framework explicitly tailored for crafting intricate user interfaces and views. Vue amalgamated the best aspects of React and Angular, presenting developers with a versatile, lightweight, and potent alternative. - -## A quick comparison: React vs Angular vs Vue - -As we embark on our journey through the world of React, Angular, and Vue, it's crucial to grasp the fundamental differences that set these popular JavaScript frameworks apart. Although they share common ground in building sophisticated and interactive web applications, each framework exhibits unique attributes that shape the developer experience. It’s useful to provide a quick comparison of React, Angular and Vue, laying the foundation for a more in-depth exploration of their technical nuances in the sections to follow. - -![](assets/choosing-the-right-javascript-framework-a-deep-dive-into-react-vs-angular-vs-vue_fca70866d812e626a871c73732276b8b_md5.webp) - -| **Feature** | **React** | **Angular** | **Vue** | -| --------------------------- | --------------------------------- | ------------------------- | ------------------------ | -| Release Year | 2013 | 2016 (Angular 2) | 2014 | -| Latest Version | 18.x | 15.x | 3.x | -| Popularity (GitHub Stars) | 200k+ | 87k+ | 210k+ | -| Developed By | Facebook | Google | Evan You | -| Programming Language | JavaScript/TypeScript | TypeScript | JavaScript/TypeScript | -| Learning curve | Moderate | Steep | Easy | -| Community Support | Large | Large | Large | -| Performance | High | High | High | -| Scalability | High | High | High | -| Component Architecture | Yes | Yes | Yes | -| CLI Tools | Create React App | Angular CLI | Vue CLI | -| Data Binding | One-way | Two-way | One-way | -| Routing | React Router | Angular Router | Vue Router | -| State Management | Redux, MobX, Zustand, Context API | RxJS, NgRx, Akita | Vuex, Pinia | -| SSR (Server-Side Rendering) | Next.js, Remix, Razzle | Angular Universal | Nuxt.js, Quasar | -| Mobile Development | React Native | Ionic, NativeScript | Vue Native, Quasar | -| UI Component Libraries | Material-UI, Ant Design, Chakra | Angular Material, Clarity | Vuetify, Quasar, Element | -| Job Market Demand | High | Moderate | Moderate | - -_Please note that the information provided in this table is an approximation and subject to change over time. It is always a good idea to keep up to date with the latest developments in these frameworks, as features and popularity may evolve._ - -## Detailed comparison: React vs Angular vs Vue - -### Rendering performance - -**React**: React is renowned for its virtual DOM, which enables it to efficiently update and render components by comparing the changes with the actual DOM. This results in excellent rendering performance, particularly in large-scale applications. Additionally, React incorporates [concurrent rendering](https://react.dev/blog/2022/03/29/react-v18#what-is-concurrent-react), which further optimizes the rendering process by prioritizing updates based on their importance. - -**Angular**: Angular utilizes a real DOM and leverages [ahead-of-time](https://angular.io/guide/aot-compiler) (AOT) compilation and change detection mechanisms to optimize rendering performance. These features, combined with zone.js for efficient change detection, have significantly improved Angular's performance over time. However, it can still struggle with complex applications, particularly when dealing with frequent DOM updates. - -**Vue**: Vue amalgamates the best of both worlds by using a virtual DOM for efficient updates and reactive data binding for quick DOM manipulations. Its lightweight nature makes it particularly performant in smaller applications, but it scales well to larger projects too. Vue 3 introduced the [Composition API](https://vuejs.org/guide/extras/composition-api-faq), which further improves performance by facilitating more granular control over-reactivity and optimization. - -### Learning curve - -**React**: React has a moderate learning curve due to its unique JSX syntax and functional programming concepts, such as hooks and higher-order components. However, the vast community support, extensive documentation, and popularity in the industry make it easier to find resources and learn quickly. React's flexibility also allows developers to adopt different patterns and architectures based on their preferences. - -**Angular**: Angular has a steeper learning curve, especially for beginners, as it involves understanding concepts like dependency injection, decorators, and TypeScript. Its comprehensive nature as a complete framework might require more time to become proficient. However, Angular's strict structure and best practices can help developers maintain a consistent codebase in large projects. - -**Vue**: Vue is known for its gentle learning curve, with an approachable API and clear documentation. It allows developers to integrate with existing projects incrementally and offers a smooth transition to a fully-featured framework. Its template syntax is familiar to developers with HTML, CSS, and JavaScript experience, making it more accessible to newcomers. - -### Tooling and ecosystem - -**React**: React boasts a mature and extensive ecosystem, with a wide array of libraries and tools available for every need. Examples include Redux and MobX for state management, React Router and Reach Router for navigation, and Create React App and Next.js for project scaffolding. Additionally, React has a large and active community that consistently contributes to the growth and improvement of the ecosystem. - -**Angular**: Angular's ecosystem is built around its CLI tool (Angular CLI) and includes built-in solutions for routing, forms, and HTTP requests. The Angular community has developed numerous component libraries and tools, such as Angular Material for UI components, Angular Universal for server-side rendering, and Nrwl Nx for monorepo development. Angular's robust ecosystem offers a cohesive experience for developers. - -**Vue**: Vue's ecosystem is growing rapidly, with the official Vue CLI, Vue Router, Vuex for state management, ViteJS for build tooling. There are also several popular component libraries like Vuetify, Quasar, and Element. While Vue's ecosystem may not be as extensive as React's, it is continually expanding and improving. The Vue community is also known for its welcoming and supportive atmosphere. - -### API stability - -**React**: React's API has remained relatively stable over the years, with Facebook's commitment to providing smooth migration paths for major updates. Backward compatibility is a priority, minimizing breaking changes for developers. The introduction of hooks in React 16.8 was a significant addition, but it was designed to be fully compatible with existing class components. - -**Angular**: Angular has experienced more significant API changes since its inception, particularly with the transition from AngularJS to Angular 2. However, Google has since adopted a predictable release schedule with clear update guidelines, ensuring more stability for developers. Angular now follows semantic versioning, which makes it easier to understand the scope of changes between versions and reduces the risk of breaking changes. - -**Vue**: Vue has maintained a stable API throughout its development, with only minor breaking changes between major versions. The Vue team prioritizes backward compatibility and provides detailed migration guides when necessary. [The transition from Vue 2 to Vue 3](https://v3-migration.vuejs.org/), for example, was accompanied by a dedicated migration build to help developers identify and address compatibility issues. - -### Real-world use cases and industry adoption - -**React**: React is widely adopted by large corporations and startups alike, including Facebook, Instagram, Airbnb, and Netflix. Its versatility and performance make it suitable for a wide range of applications, from social media platforms to e-commerce websites, enterprise-level solutions, and even virtual reality experiences with React VR. - -**Angular**: Angular has a strong presence in enterprise environments, with companies like Google, Microsoft, IBM, and Forbes utilizing the framework for their applications. Its comprehensive feature set and built-in tooling make it ideal for complex, large-scale applications and enterprise-level solutions. Angular is also commonly used in financial services, healthcare, and government projects, where its robustness and scalability are highly valued. - -**Vue**: Vue has seen increasing adoption in various industries, with companies like Alibaba, Xiaomi, and GitLab using it for their projects. Its ease of use and flexibility make it suitable for both small and large-scale applications, ranging from single-page applications to advanced web applications and progressive web apps. Vue's growing popularity has also led to its adoption in open-source projects and community-driven initiatives. - -## Conclusion - -In conclusion, React, Angular, and Vue each have their strengths and weaknesses, and the choice of framework ultimately depends on your project's specific requirements and your personal preferences as a developer. React excels in performance, scalability, and community support, making it a go-to choice for many developers. Angular is a comprehensive framework with powerful built-in tooling, making it a favorite among enterprise-level applications. Vue, on the other hand, offers a gentle learning curve and flexible approach, making it an attractive option for both beginners and experienced developers. By understanding the differences and unique aspects of each framework, you can make an informed decision and select the best tool for your next web development project. diff --git a/topics/frontend/concurrency-in-javascript.md b/topics/frontend/concurrency-in-javascript.md deleted file mode 100644 index 2cc6b7a7a..000000000 --- a/topics/frontend/concurrency-in-javascript.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -title: Concurrency in javascript -description: Learn how JavaScript’s single-threaded event loop enables asynchronous, non-blocking concurrency using callbacks, Promises, and async/await to handle tasks efficiently and avoid callback hell. -date: 2022-08-26 -github_id: hieuphq -redirect: - - /7ge9Sg ---- - -Javascript is a single-threaded programming language that has single-threaded at the runtime. It has a single call stack so it can do one thing simultaneously. JavaScript has a runtime model based on an **event loop**, which is responsible for executing the code, collecting and processing events, and executing queued sub-tasks. Javascript is a single-threaded, non-blocking asynchronous concurrent language. A JavaScript runtime uses a message queue, a list of messages to be processed. Each message has an associated function that gets called to handle the message. - -```javascript -while (queue.waitForMessage()) { - queue.processNextMessage(); -} -``` - -Each message is processed completely before any other message is processed. There are three components in the runtime concepts: stack, heap, and queue. - -- Stack: contains the function calls. Each function call creates a stack frame that includes the statement and expression in the function. When that function returns a value or void, its frame is popped out, and the next function will begin executing. If we call a function inside another function, a frame will be created for each. -- Heap: is a largely unstructured area of memory allocated to the objects in our code. -- Callback queue: is a data structure, message queue, that stores a list of messages to be processed. Each message has an associated function that gets called to handle the message. - -```javascript -function mult(a, b) { - return a * b; -} -function square(n) { - return mult(n, n); -} -function printSquare(n) { - let squared = square(n); - console.log(squared); -} - -printSquare(4); -``` - -When running the above program, the Javascript will run the function step-by-step and add the function call to the `stack`. The first is: that the `printSquare()` function is added to the stack. The next functions are `square()`, `mult()`. The items in the `stack` will be executed orderly. - -## Javascript is a single-threaded language - -In reality, we always execute many tasks for a long time. These are opening a big file, downloading an image, or invoking a third-party service... We need to wait for the function call to be completed. How do we deal with the function call with the execution time too long? The Javascript runtime that is supported from the web browser, node.js. The web browser, or JS engines,... provides the APIs that help us run the task outside the `stack`. It means the long-time functions have already been finished in the stack, and the following function in the `stack` won't be blocked. - -![](assets/concurrency-in-javascript_javascript-concurrency-model.webp) - -After the JS engine executes the task, it will push the callback onto the `callback queue`. If the `stack` is empty, it takes the first thing on the `callback queue` and makes it onto the `stack`. In conclusion, JavaScript code is single-threaded and only does one thing at a time; however, the **Javascript runtime** executes the task asynchronously as multi-threaded. Once that background thread completes, and the current call stack finishes executing, your callback function is pushed onto the (now empty) call stack and run to completion. However, the developer faced a new issue called Callback Hell. Which is essentially nested callbacks stacked below one another, forming a pyramid structure. Every callback depends/waits for the previous callback, thereby making a pyramid structure that affects the readability and maintainability of the code. We solved the problem by the data structures or built-in functions. - -## Implement using callback in reality - -We can implement a feature called `payMyBill()` that will read the invoice total from the backend, call the third-party service API to charge money, update the billing status, and update the cart. Assuming the `getInvoiceTotal()`, `chargeMoney()`, `updateBilling()`, and `updateCart()`. - -```javascript -let getInvoiceTotal = (invoiceId, callback) => { - let invoiceData = fetch("/invoice/#{invoiceId}"); - callback(invoiceData); -}; -let chargeMoney = (total, callback) => { - rs = chargeAPI("/charges/#{total}"); - callback(rs); -}; -let updateBilling = (data, callback) => { - let res = fetch("PUT", "/invoice/#{invoiceId}", data); - callback(res); -}; -let updateCart = (data, callback) => { - Cart.update(data); - callback(true); -}; - -let payMyBill = (invoiceId) => { - return getInvoiceTotal(invoiceId, function (invoice) { - chargeMoney(invoice.total, function (isSuccess) { - updateBilling(isSuccess, function (updated) { - updateCart(updated, function (cart) { - return cart; // callback hell - }); - }); - }); - }); -}; -``` - -## Promise - -A Promise is a Javascript object that allows us to make async calls. It produces a value when the async operation completes successfully or an error if it doesn't complete. A Promise object has two main components: state of the execution and result. - -- Pending: when the execution function starts. -- Fulfilled: when the promise resolves successfully. -- Rejected: when the promise rejects or gets an error. - -```mermaid -stateDiagram-v2 -    [*] --> Pending -    Pending --> Fulfilled -    Pending --> Rejected -    Fulfilled --> [*] -    Rejected --> [*] -``` - -In the `promise` implementation, we can find out they use `setTimeout()` function. It ensures the task is always running asynchronously. We can use the Promise chaining to flatten the logic of the callback chain. - -```javascript -let getInvoiceTotal = (invoiceId) => {...}; // return the Promise -let chargeMoney = (total) => {...}; // return the Promise -let updateBilling = (data) => {...}; // return the Promise -let updateCart = (data) => {...}; // return the Promise - -let payMyBill = (invoiceId) => { - return getInvoiceTotal(invoiceId) - .then(invoice => chargeMoney(invoice.total)) - .then(isSuccess => updateBilling(isSuccess)) - .then(updated => updateCart(updated)) - .then(cart => { - return cart; - }); -}; -``` - -## async and await - -ES2017 introduced the `async`/`await` that helps you write the code cleaner than the promise chaining technique. - -- `async` keyword allows us to define a function that handles asynchronous operations. -- `await` keyword to wait for a `Promise` to settle either in the resolved or rejected state. - -```javascript -let payMyBill = async (invoiceId) => { - let invoice = await getInvoiceTotal(invoiceId); - let isSuccess = await chargeMoney(invoice.total); - let updated = await updateBilling(isSuccess); - let cart = await updateCart(updated); - - return cart; -}; -``` - -To get the most out of our single-threaded program, we need to invoke JavaScript’s event loop superpowers. We can queue two async operations and wait for both to complete. In our example, we should make `updateBilling` and `updateCart` run simultaneously. We used `Promise.all()` to wait for concurrent operations to finish, then aggregated the results to update the `Cart`. The `Promise.all()` function works just fine for a few concurrent spots, but code quickly devolves when you alternate between chunks of code that can be executed concurrently and others that are serial. This intrinsic ugliness is not much improved with `async` functions. - -## Race condition - -The term "race condition" usually refers to a conflict when accessing shared variables in a multithreading environment. Although your Javascript code is only executed by one thread at a time, it is still possible to encounter similar problems. This is a common issue when people make their functions 'async' without thinking about the consequences. The best way to avoid these issues is to prevent using async functions when they are not necessary. A pure function is described as: - -- A referentially transparent `function` will always return the same output for the same input. -- Have no side effects - it doesn't affect the outside world. The referential transparency and the lack of side effects make pure functions more applicable. Other benefits are better composability, unit testing, parallelization, easier debugging, etc. - -## Conclusion - -Concurrent code is preferable to sequential code because it is non-blocking and can handle several events simultaneously. The JavaScript’s event loop helps us solve the concurrency problem. We can handle shared states by combining results inside a single thread. The new syntax or library supported JS developers' coding experience. We solved callback hell by Promise; `async`/`await` keyword. Remember, _JavaScript is a single-threaded language and, at the same time, also non-blocking, asynchronous, and concurrent_ - -## References - -https://developer.mozilla.org/en-US/docs/Web/JavaScript/EventLoop https://www.promisejs.org/implementing/ https://www.geeksforgeeks.org/what-to-understand-callback-and-callback-hell-in-javascript https://javascript.info/promise-chaining https://www.javascripttutorial.net/es-next/javascript-async-await/ diff --git a/topics/frontend/continuous-translation.md b/topics/frontend/continuous-translation.md deleted file mode 100644 index bdf9680f1..000000000 --- a/topics/frontend/continuous-translation.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -title: Continuous translation -description: Continuous translation (CT) is a modern approach to translation management that involves synchronizing software development and translation workflows. This means that developers, translators, and product owners work together in a continuous cycle to ensure that all translations are up-to-date and aligned with the latest software developments. -date: 2023-04-11 -authors: - - tienan92it -tags: - - translation - - engineering - - web -redirect: - - /Guzjvw ---- - -![](assets/continuous-translation_eaf982ca480d0677ec9d8fd34553b51a_md5.webp) - -Continuous translation (CT) is a modern approach to translation management that involves synchronizing software development and translation workflows. This means that developers, translators, and product owners work together in a continuous cycle to ensure that all translations are up-to-date and aligned with the latest software developments. - -Unlike traditional translation workflows, Continuous translation eliminates the need for file and space synchronization between different stakeholders, streamlining the translation process and promoting better collaboration. By implementing Continuous translation, companies can achieve faster development cycles, higher-quality translations, and a better user experience for their customers. - -## Working with localization - -### How localization generally works - -![](assets/continuous-translation_0ca3440b9dbd840d349d587dd7fd6a1d_md5.webp) - -The classical or typical approach in any software project will have translated data be coupled with the repository (plus its software release) and its access to the data. This data could exist in a database or directly as a file. Changes to the data would be dependent on the coupling of the git repository and the translation management system, which becomes a blocker for translation teams. - -### Alternative solutions to localization - -In order to reduce the friction between the developing team and the translation team, we can remove data and process coupling between systems of both teams by centralizing the data to allow for continuous translation. In this article we’ll discuss 2 possible solutions to integrate continuous translation into our current system: - -- **Solution 1**: implement an ad-hoc solution ourselves with help from Google Sheets (using Google Translator under the hood). -- **Solution 2**: translation as a service - we use a third party service for manage our translations. - -### Solution 1: Google Sheets - -Google Sheets is a free and easy-to-use tool that allows us to manage translations with all stakeholders. Before deploying the app, developers can use Git actions to fetch the latest version of the Google Sheets and generate the locale translation files. - -Here are what the steps to use Google Sheets would look like for managing translations: - -**Step 1:** Create a Google Sheet that contains all supported translation items. Here is an example **[template](https://docs.google.com/spreadsheets/d/1jjVDCMAmS6WySmB7L25yNCZX-X3jEwrrqLhqOBzVEs0/edit?usp=sharing)** you can use. **Step 2:** Fetch data from Google Sheets using the following code: - -_Make sure to replace the _`***sheetId***`_ and _`***sheetName***`_ with your own values_ - -```javascript -// Save on './public/spreadsheet.ts' file -const sheetId = ""; -const sheetName = ""; -const baseUrl = `https://opensheet.elk.sh/${sheetId}/${sheetName}`; - -// Path to store all translation data -const translateDataPath = "./public/locales/translate-data.json"; - -export const getJsonData = async () => { - const res = await fetch(baseUrl, { - method: "GET", - headers: { - "Content-Type": "application/json", - accept: "*/*", - authority: "opensheet.elk.sh", - }, - mode: "cors", - credentials: "omit", - }); - - if (res.ok) { - return await res.json(); - } -}; - -getJsonData().then((data) => { - const fs = require("fs"); - let myObject = data; - - // Writing to our JSON file - var newData = JSON.stringify(myObject, null, 2); - fs.writeFile(translateDataPath, newData, (err) => { - // Error checking - if (err) throw err; - console.log("New data added"); - }); -}); -``` - -**Step 3: **After having obtaining the translation data, generate locale files for supported languages using the following code: - -```javascript -// Save on './public/manage-translations.ts' file -import path from "path"; -import fs from "node:fs/promises"; -import fsExtra from "fs-extra"; -import _ from "lodash"; - -import dotenv from "dotenv"; -dotenv.config({ path: ".env.local" }); -dotenv.config({ path: `.env.${process.env.NODE_ENV}` }); -dotenv.config(); - -const validLang = ["en-US", "de-DE", "fr-FR"]; -const defaultLanguage = - process.env.LOCALE == null ? "en-US" : validLang.includes(process.env.LOCALE) ? process.env.LOCALE : "en-US"; - -const configs = { - defaultLanguage, - otherLanguages: validLang.filter((lang) => lang !== defaultLanguage), - rootExportPath: "./public/locales", -}; - -const allLocales = []; - -async function generateJSONFiles() { - const data = await fs.readFile("./public/locales/translate-data.json", "utf8"); - const jsonData = JSON.parse(data); - const locales = Object.keys(jsonData[0]).filter((key) => key !== "elementId"); - const result = locales.map((locale) => { - const data = {}; - jsonData.forEach((item) => { - data[item.elementId] = -``` - -### Solution 2: Use a translation management platform - -For this approach, we can use [Locize](https://locize.com/) as our continuous localization management platform. This approach isn’t limited to Locize, but the idea is to have a platform to decouple software release from the translation work and minimize work friction for translation. - -![](assets/continuous-translation_5d971ebb8af780ed2ad9e7626daf0d8c_md5.webp) - -Locize has integration support for a variety of frontend systems. You can integrate Locize by following steps: - -- Follow [https://docs.locize.com/integration/getting-started](https://docs.locize.com/integration/getting-started) to create a Locize account and project. -- After the project is created, we will have `project id` and `api key`. -- Use [https://github.com/locize/locize-cli](https://github.com/locize/locize-cli) to synchronize the existing translations with Locize - -There are three ways to use `Locize` in your app: - -- **Approach 1**: Use Locize live download on the client-side only. This option involves bundling translations in your app to prevent an elevated amount of downloads on the server-side. Before deploying your app, synchronize your translations with Locize so that they are bundled in your app. This way, your server-side will not generate any downloads to the Locize CDN during runtime, but only on the client-side. -- **Approach 2:** Configure Locize to download translations live on both client (browser) and server (node.js). - -> Do not use this option if you have a `serverless environment` as it can generate too many download requests and run up your bill. - -- **Approach 3:** Bundle translations with your app. This option involves bundling translations in your app at build time. It's recommended to use this option if you have a small number of translations or if your translations don't change frequently. - -### Comparisons between solutions - -**Solution 1** recommends using a custom translation file and providing translations for each language in a separate JSON file. The translation file is then loaded on the server side and used to render the content in the appropriate language. This solution is relatively simple and straightforward to implement, but it can become cumbersome to manage as the number of languages and translations grows. - -**Solution 2** proposes using a translation management platform, with one example using Locize, which allows for continuous localization management. This solution involves integrating Locize into the application, synchronizing the existing translations with Locize, and then bundling the translations in the application using one of three different possibilities, depending on the specific use case. This solution requires more setup and configuration but can provide a more scalable and streamlined approach to managing translations. - -In summary, `Solution 1` is a simpler and cheaper approach to manage translations, while `Solution 2` is a more advanced solution that provides more flexibility and scalability in managing translations. - -### Pros and cons when integrating Continuous translation into your app - -**Pros** - -- **Faster development cycles**: With Continuous translation, developers and translators work together in a continuous cycle, ensuring that translations are updated in real-time as new features are developed. This leads to faster development cycles and quicker time-to-market. -- **Improved translation quality**: Continuous translation promotes better collaboration between developers and translators, which can lead to higher-quality translations that accurately reflect the intended meaning of the original content. -- **Better alignment between teams**: By eliminating file and space synchronization issues, Continuous translation helps to align developers, translators, and product owners more closely, reducing communication errors and promoting better collaboration. -- **Reduced costs**: Traditional translation workflows can be time-consuming and costly. By streamlining the translation process, Continuous translation can help to reduce translation costs and improve return on investment. - -**Cons** - -- **Requires significant coordination**: Continuous translation requires a high level of coordination between different teams, including developers, translators, and product owners. This can be challenging to manage, particularly for larger projects. -- **Potential for errors**: Continuous translation requires real-time updates to translations, which can increase the risk of errors and miscommunication. This requires careful management and quality control. -- **Requires specialized tools**: Implementing Continuous translation requires specialized tools and technologies, which can add to the overall cost of the project. -- **Not suitable for all projects**: Continuous translation may not be suitable for all projects, particularly those with limited budgets or resources. Traditional translation workflows may be more appropriate for smaller projects or those with less frequent updates. - -## Conclusion - -Multilingual support is a very important function for web or mobile applications nowadays. Users will come from all over the world and always ask for support for their language. The two options above have different advantages and disadvantages, so you need to consider the exact scope of the product to have the best choice for your team. Both methods can meet the needs of constantly translating products to support new features or new products, but it will cost production as well as quality assurance. diff --git a/topics/frontend/css-container-queries.md b/topics/frontend/css-container-queries.md deleted file mode 100644 index c5e8538b0..000000000 --- a/topics/frontend/css-container-queries.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Css container queries -description: Container Queries is a CSS feature that allows us to style elements based on the size of a container. -date: 2022-09-02 -authors: - - tienan92it -github_id: tienan92it -tags: - - css - - frontend - - guide - - responsive-design -redirect: - - /RAsm8w ---- - -Published on August 30, 2022, Chrome 105 included Container Queries, one of the most highly requested features in CSS. - -## The problem with media queries - -When we want to create a responsive layout, we can use media queries to adjust styles based on the screen size of the device viewing our site. However, media queries have some limitations. For example, we cannot use them to style individual components based on their parent's width; we can only adjust the entire page. This is problematic because we cannot create responsive components; only responsive pages. - -## What is Container Queries? - -The CSS3 property "container queries" allows us to style elements based on the size of a container. It is similar to a Media Query, except it evaluates against the size of a container instead of the size of the viewport. - -## How to use Container Queries? - -To query a component based on its parent width, we need to use the `container-type` property with possible values: `size`, `inline-size`, `block-size`, `style`, `state`. - -```css -.sidebar { - container-type: inline-size; -} -``` - -Now we can start to query a container using `@container`. This will query the nearest containment context. - -```css -@container (min-width: 300px) { - .content { - display: none; - } -} -``` - -Additionally, Containers can be named with `container-name` property. This allows us to query a specific container. - -```css -.sidebar { - container-type: inline-size; - container-name: my-sidebar; -} -``` - -or shorthand syntax - -```css -.sidebar { - container: my-sidebar / inline-size; -} -``` - -Then, to query a specific container, we can use `@container` with `container-name`. - -```css -@container my-sidebar (min-width: 300px) { - .content { - display: none; - } -} -``` - -## Browser support - -Currently, Container Queries is only available in [modern browsers](https://caniuse.com/?search=Container%20Queries): - -- Chrome 105+ -- Safari 15+ -- Edge 105+ - -Universal support can be achieved by using [Polyfill](https://github.com/GoogleChromeLabs/container-query-polyfill). - -## Reference - -- https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Container_Queries -- https://javascript.plainenglish.io/css-container-queries-3393fbeb6ea8 -- https://ishadeed.com/article/container-queries-are-finally-here/ -- https://developer.chrome.com/docs/devtools/css/container-queries/ diff --git a/topics/frontend/css-in-js.md b/topics/frontend/css-in-js.md deleted file mode 100644 index 4a66611a3..000000000 --- a/topics/frontend/css-in-js.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: Css in JS -description: CSS-in-JS is a styling technique where Javascript is used to style the component. -date: 2022-08-11 -authors: - - trankhacvy -github_id: trankhacvy -tags: - - css-in-js - - frontend -redirect: - - /XTNRfQ ---- - -**CSS-in-JS** is a styling technique where Javascript is used to style the component. These are multiple implementations of this concept in the form of libraries such as [emotion](https://emotion.sh), [styled-component](https://styled-components.com/), [JSS](https://cssinjs.org). They aim to tackle the limitations of CSS, such as lack of dynamic functionality, scoping, and portability. - -## Benefits - -### Scoping - -With CSS-in-JS, writing new styles cannot affect anything else in other places on the site, so there’s no need to worry about writing a style that has bad or unintended consequences elsewhere due to a selector in the global scope. - -### Avoid naming collisions - -[Naming is hard](https://hilton.org.uk/blog/why-naming-things-is-hard). CSS-in-JS libraries automatically generate unique selectors for what’s being styled. We don't need to think about naming. - -### Dynamic functionality - -As CSS-in-JS is essentially JavaScript code, you can apply complex logic to your style rules, such as loops, conditionals, variables, state-based styling, and more. - -### Dead code elimination - -CSS-in-JS helps with removing dead code. The only styles that are loaded are the styles for the components in use at any given time. There’s no shipping of any unused styles. When a component dies, so does its style. - -### Developer ergonomics - -It can be nice to have styles in the same file (or otherwise very close to) the component itself. In the same way, some developers feel very comfortable in JSX. Also, being able to style things without any scoping worry means developers may feel empowered about styling rather than intimidated by it. - -## Disadvantages - -### Runtime cost - -When CSS is generated from JavaScript at runtime, in the browser, there is an inherent overhead. Some CSS-in-JS libraries try to overcome this overhead by extracting CSS files during the build time (like [linaria](https://linaria.dev)), but it comes with other [trade-offs](https://github.com/styled-components/styled-components/issues/2377) - -### Learning curve - -CSS-in-JS definitely has a learning curve, especially if you have used neither component-based frameworks nor web components before. Besides learning the new syntax, you also need to pick up a new way of thinking, which needs time and might slow down your development workflow for a while. - -### Unreadable class names - -Automatically generated selectors significantly worsen code readability. This can be a huge concern for you if you regularly use your browser’s developer tools for debugging. Currently, many CSS-in-JS libraries try to provide meaningful class names based on the declaration name or component name in development mode. Some of them even let you customize the class name generator function. In production mode, though, the class names are still hard to read and debug. - -### Extra bundle size - -Adding another library to a web page increases the page size, which can negatively impact page load time. - -## Reference - -- https://en.wikipedia.org/wiki/CSS-in-JS -- https://webdesign.tutsplus.com/articles/an-introduction-to-css-in-js-examples-pros-and-cons--cms-33574 -- https://medium.com/dailyjs/what-is-actually-css-in-js-f2f529a2757 -- https://github.com/styled-components/styled-components/issues/2377 diff --git a/topics/frontend/debugging-in-javascript.md b/topics/frontend/debugging-in-javascript.md deleted file mode 100644 index 3c3e7a682..000000000 --- a/topics/frontend/debugging-in-javascript.md +++ /dev/null @@ -1,157 +0,0 @@ ---- -title: Debugging in javascript -description: Learn how to use JavaScript console methods like console.log, console.assert, console.dir, console.time, and the debugger statement to effectively debug code in Chrome DevTools. -date: 2022-10-16 -github_id: namtrhg -redirect: - - /JbbLGg ---- - -_This note focus on utilizing some of the **` console`** object methods and the **` debugger`** statement to better debug JavaScript application in the [Chrome DevTools](https://developer.chrome.com/docs/devtools/)._ - -## The `console` object - -The **` console`** object provides access to the browser's debugging console. You can view it by right-clicking on your Chrome browser, selecting **Inspect**, and choosing **Console** in the tab. -![](assets/debugging-in-javascript_chrome_devtools's_console_tab.webp) - -### `console.log()` - -The `console.log()` method writes to the web console. The message can be a single text (with optional replacement values) or any number of JavaScript objects. - -#### Syntax - -```js -console.log(obj1); -console.log(obj1, /* …, */ objN); -console.log(msg); -console.log(msg, subst1, /* …, */ substN); -``` - -#### Use-case - -This method can be used to print any kind of variables defined before in it or to just print any message that needs to be displayed to the user. - -```js -const name = "Dwarves Foundation"; -console.log(name); -//=> Dwarves Foundation -``` - -### `console.assert()` - -The `console.assert()` method writes a message to the console if an expression evaluates to `false`. - -#### Syntax - -```js -console.assert(_expression_, _message_); -``` - -#### Use-case - -Let's say you wanted to check for a condition of a user id, you might be checking using the `console.log()` like this: - -```js -if (!user.id) { - console.log("User does not exist!"); - // user.id = false? => User does not exist! -} -``` - -Instead, you can check the condition using the `console.assert()`: - -```js -console.assert(user.id, "User does not exist!"); -// user.id = false? => Assertion failed: User does not exist! -``` - -Using `console.assert()` provide a much cleaner and better way for conditional logging in your application. - -### `console.dir()` - -The `console.dir()` displays all of the properties of a specified JavaScript object in the console, allowing the developer to conveniently access the object's properties. - -#### Syntax - -```js -console.dir(object); -``` - -#### Use-case - -In Chrome, `console.log` prints out a tree -- _most of the time_. However, Chrome's `console.log` still stringifies certain classes of objects, even if they have properties. A regular expression is the most obvious example of a distinction: - -```js -const array = [1, 2, 3]; -console.log(array); -// [1, 2, 3] -console.dir(array); -/* Array[3] - 0: 1 - 1: 2 - 2: 3 - length: 3 - * __proto__: Array[0] - concat: function concat() { [native code] } - constructor: function Array() { [native code] } - entries: function entries() { [native code] } - ... */ -``` - -Another useful difference in Chrome exists when sending **`DOM`** elements to the console: -![console.dir() example]() - -### `console.time()` and `console.timeEnd()` - -The `console.time()` method launches a timer that you may use to track the duration of the operation. You may have up to 10,000 timers running on a single page, giving each one a unique name. When you use `console.timeEnd()` with the same name, the browser returns the time in milliseconds since the timer was started. - -#### Syntax - -```js -console.time(label); -``` - -#### Use-case - -These methods can be used to calculate how much time a function takes to run. - -```js -console.time("foo"); -function Foo() { - //do something -} -console.timeEnd("foo"); -//=> foo: 0.00... ms -``` - -## The `debugger` statement - -The `debugger` statement activates any debugging capability available, such as setting a breakpoint. This statement has no impact if no debugging functionality is present. - -#### Syntax - -```js -debugger; -``` - -#### Use-case - -The `debugger` statement can be use when you want to check the behavior of a potentially buggy function. - -```js -function potentiallyBuggyFunction() { - debugger; - // do potentially buggy stuff to examine, step through, etc. -} -``` - -When the `debugger` is invoked, execution is paused at the `debugger` statement just like a breaking point. -![](assets/debugging-in-javascript_debugger_example.webp) - -## Reference - -- [Debugger's statement - Developer Mozilla Organization](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger?retiredLocale=vi) -- [Console object - Developer Mozilla Organization](https://developer.mozilla.org/en-US/docs/Web/API/console) -- [What is assert in JavaScript? - StackOverflow](https://stackoverflow.com/questions/15313418/what-is-assert-in-javascript) -- [What is the difference between console.dir() and console.log()? - StackOverflow](https://stackoverflow.com/questions/11954152/whats-the-difference-between-console-dir-and-console-log) -- [Tips and tricks for debugging JavaScript - James Q Quick](https://www.youtube.com/watch?v=_QtUGdaCb1c&t=701s) diff --git a/topics/frontend/deno.md b/topics/frontend/deno.md deleted file mode 100644 index f8adc1a96..000000000 --- a/topics/frontend/deno.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Deno -description: Discover Deno, a secure JavaScript runtime built in Rust with native TypeScript, WASM, and ESM support, designed for modern web APIs and easy deployment without complex build tools. -date: 2022-03-17 -github_id: huygn -redirect: - - /Jqv03w ---- - -## What - -- A new JavaScript runtime, written in Rust (JsVM 💃🏻) -- Built to follow the standard Web API -- Take advantage of TypeScript, WASM and ESM module (and support them natively) -- "Node" written weirdly backwards - -## Why - -![](assets/deno_kfo8ecl.webp) - -- Deno follows the standard Web API [and is leading the industry in implementing them](https://github.com/denoland/deno/pull/11941) - - This is with the aim to make JS/TS project ideally more universal -- secure by default - - requires `--allow-*` flags to enable specific features like read/write, network access, etc... - - This is so a random project/module can't read your file system unless you **explicitly** allow it to -- no need for `npm install`, pre-bundling (e.g: `tsc`/`webpack`/`package.json`) dances and rituals -- Perfect to deploy on modern platforms like Cloudflare Workers and [Deno deploy](https://deno.com/deploy/docs) - -## Why not? - -- Although native ESM and `script type="module"` is widely supported in modern browsers, some very new APIs like `URLPattern` is not completely supported everywhere -- dependencies on Deno are imported by URL, so it makes it somewhat messy, but [`import map`](https://deno.land/manual/linking_to_external_code/import_maps) can be used to improve readability - - `import lodash from "https://deno.land/x/lodash@4.17.19"` - - `deno` will download available imports online and cache locally before running - -## What about Node.js? - -- Node.js and its ecosystem has grown into a messy place -- History of inconsistent authoring/bundling [formats](https://dev.to/iggredible/what-the-heck-are-cjs-amd-umd-and-esm-ikm) (cjs, amd, umd) -- Node 16 adds ESM support, but before moving to ESM format, 2 things needed to happen: - - the library author has to [break backward compatibility](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#pure-esm-package) - - library users have to [migrate their project to an ESM format](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#pure-esm-package) - - just imagine the whole Node.js ecosystem going through this 💥 -- Confusion with JavaScript in the browser (e.g: `require` vs `import`) -- Non-standard & incompatible with modern JavaScript - -## Extras - -Deno also removes one most confusing part when developing JavaScript: build tools. - -Webpack/Babel and their bundling process is the most over-engineered (albeit necessary) part of JavaScript development. Just needing to know how bundlers work to effectively leverage them is a crucial part of writing & authoring JavaScript projects/libraries, and has been over the recent decade. - -Node.js developers often need to work with Webpack & Babel just to have the convenience to write their code in modern JavaScript (e.g: `import` vs `require`) or be able to work with TypeScript. - -Deno, on the other hand, strives to move along with the modern JavaScript standard by defaulting to ESM, with native support for all standard APIs and TypeScript/JSX, removing the need to rely on build tools. - -> "Now we can sit down and write actual JavaScript instead of spending half a day initialize new project with Webpack, Babel, JSX and another half to configure Jest to read our `.babelrc` config. _Such. fresh. air._" - -### Deno deploy - -Similar to Cloudflare Workers, Deno deploy is a serverless deployment platform using the Service Worker API to create deployments, currently in (free) public beta. - -What's really cool about Deno deploy is that it only requires a link to your entry file to deploy, and deploy time is super fast. - -![](assets/deno_gygtlws.webp) - -Interesting projects to follow: - -- https://github.com/exhibitionist-digital/ultra -- https://github.com/lucacasonato/fresh diff --git a/topics/frontend/focus-trap.md b/topics/frontend/focus-trap.md deleted file mode 100644 index 011f19973..000000000 --- a/topics/frontend/focus-trap.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Focus trap -description: Focus trap is a web technique to improve accessibility by forcing keyboard focus inside a DOM element, to prevent the user navigating outside that element. -date: 2022-06-12 -authors: - - zlatanpham -github_id: zlatanpham -tags: - - accessibility - - frontend - - html -redirect: - - /0NURcA ---- - -Focus trap is a web technique to improve accessibility by forcing keyboard focus inside a DOM element, to prevent the user navigating outside that element. This technique normally applies for an open modal dialog, when user interaction is trapped inside the dialog. This makes sense because allowing the focus to escape outside the modal would possibly create a state where the user doesn't know which element is in focus. - -#### Reference - -- https://css-tricks.com/a-css-approach-to-trap-focus-inside-of-an-element/ diff --git a/topics/frontend/from-markup-to-pixels-a-look-inside-the-dom-cssom-and-render-tree.md b/topics/frontend/from-markup-to-pixels-a-look-inside-the-dom-cssom-and-render-tree.md deleted file mode 100644 index f6fe52fec..000000000 --- a/topics/frontend/from-markup-to-pixels-a-look-inside-the-dom-cssom-and-render-tree.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: From markup to pixels - a look inside the DOM, CSSOM, and render tree -description: A deep dive into the rendering process of a web page, exploring the Document Object Model (DOM), CSS Object Model (CSSOM), and the Render Tree. -date: 2023-09-11 -authors: - - nguyend-nam -github_id: nguyend-nam -tags: - - cssom - - dom - - render-tree - - rendering -redirect: - - /WG6bbQ ---- - -Inside a web browser, there exists a software component responsible for determining the content to show you based on the data it receives. This component is known as the browser engine. - -The browser engine is a fundamental element present in all major web browsers, and various browser manufacturers give their engines distinct names. For instance, Firefox's browser engine is known as [Gecko](https://en.wikipedia.org/wiki/Gecko_(software)), while Chrome uses [Blink](https://en.wikipedia.org/wiki/Blink_(browser_engine)), a derivative of [WebKit](https://en.wikipedia.org/wiki/WebKit). - -## From raw bytes of HTML to DOM - -We are all familiar with the term **Document Object Model** or DOM, but it's always helpful to revisit how this crucial component is constructed: It starts with **bytes** of data that the browser reads from the HTML files, which are then transformed into readable **characters**. - -But that bunch of text doesn't produce an actual website. Those characters are parsed into **tokens**, such as the start or end HTML tags, the content characters and so on. The browser then organizes those tokens into **nodes**, and assembles into the hierarchical structure known as the DOM. - -> Check out this article for a more detailed understanding about this process: [How web browsers work - parsing the HTML (part 3, with illustrations)](https://dev.to/arikaturika/how-web-browsers-work-parsing-the-html-part-3-with-illustrations-45fi). - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_html-parser.webp) - -_Image Source: [How web browsers work - parsing the HTML (part 3, with illustrations)](https://dev.to/arikaturika/how-web-browsers-work-parsing-the-html-part-3-with-illustrations-45fi)_ - -## How about CSS? - -As soon as the browser begins to parse the HTML, once encountering a `link` tag to a CSS file, it simultaneously makes a request to fetch that. As you may anticipate, the procedure mirrors the way browsers form the DOM (from bytes to the object model). This process forms a tree structure called the **CSS Object Model** (CSSOM). - -> For further information, please check out this article: [How web browsers work - parsing the CSS (part 4, with illustrations)](https://dev.to/arikaturika/how-web-browsers-work-parsing-the-css-part-4-with-illustrations-4c). - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_cssom.webp) - -_Image Source: [How web browsers work - parsing the CSS (part 4, with illustrations)](https://dev.to/arikaturika/how-web-browsers-work-parsing-the-css-part-4-with-illustrations-4c)_ - -As you can see in the example above, the elements have both inherited styles from the parent (depicted in white), and their own styles that overwrite the inherited ones (depicted in black). And since we can have several CSS files linked to our HTML, that's when the [Specificity](https://developer.mozilla.org/en-US/docs/Web/CSS/Specificity) comes in handy for the browser in determining which style to apply to a particular node. - -## Combining the DOM with the CSSOM - -The browser traverses every visible node starting from the root of the DOM tree. Certain nodes are ignored by default (such as script or meta tags), while for others, the browser seeks the relevant rules within the CSSOM and proceeds to paint them on the screen. The browser will also ignores the nodes that are hidden due to their CSS property (`display: none` for example). - -Now the Render Tree holds the information of the **visibility** of each node and its **styles**, but what needs to be done next to calculate the exact position of those elements and paint them to the screen? - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_render-tree.webp) - -_Image Source: [How web browsers work - the render tree (part 7, with illustrations)](https://dev.to/arikaturika/how-web-browsers-work-the-render-tree-part-7-with-illustrations-24h3)_ - -## The layout/reflow stage - -The **Layout**/**Reflow** is a process to find the **geometry** of elements. The main thread walks through the DOM and constructs a layout tree that stores details such as **x and y coordinates**, as well as **bounding box dimensions**. - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layout-tree.webp) - -_Image Source: [Inside look at modern web browser (part 3)](https://developer.chrome.com/blog/inside-browser-part3/)_ - -This process happens every time we change something in the DOM that affects the layout, which may include: - -- Adding or deleting elements from the DOM -- Resizing the browser window -- Changing the width, the position of an element or floating it - -## The painting stage - -Despite having a DOM, style information, and a layout, these elements alone do not suffice for rendering a web page. Once the browser determines which nodes should be visible and calculates their positions within the viewport, the next step is to actually draw (render) them. Similar to the layout phase, the painting process doesn't occur just once but repeatedly whenever there are changes to the appearance of on-screen elements. - -**Painting** means the browser needs to draw every visual part of an element to the screen, including text, colors, borders, shadows, and elements like buttons and images. To ensure repainting can be done even faster than the initial paint, the drawing to the screen is generally broken down into several **layers**. When this happens, compositing becomes necessary. - -## Compositing - -**Compositing** is a method used to divide different elements of a webpage into distinct layers, independently rendering each layer, and then combining them to form a complete page in a separate thread known as the compositor thread. When scrolling occurs, this approach becomes efficient because the layers are already rendered, requiring only the composition of a new frame. Similarly, animation can be achieved by repositioning these layers and composing a new frame, enabling smooth and responsive animations. - -To determine which elements should be placed within specific layers, the main thread goes through the layout tree to generate the **Layer tree**. - -> Learn more about the **Layers panel** of DevTools [here](https://blog.logrocket.com/eliminate-content-repaints-with-the-new-layers-panel-in-chrome-e2c306d4d752/). - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_layer-tree.webp) - -_Image Source: [Inside look at modern web browser (part 3)](https://developer.chrome.com/blog/inside-browser-part3/)_ - -Once the Layer tree is established and the orders of rendering are determined, the main thread transfers that information to the compositor thread. The compositor thread then begins the process of rendering each layer. Some layers, such as those having the entire length of a webpage, can be quite extensive. To handle this, the compositor thread breaks them down into smaller sections known as **tiles** and dispatches each tile to **raster threads**. These raster threads are responsible for rendering each tile and storing the results in the memory of the GPU. - -![](assets/from-markup-to-pixels---a-look-inside-the-dom,-cssom,-and-render-tree_compositing.webp) - -_Image Source: [Inside look at modern web browser (part 3)](https://developer.chrome.com/blog/inside-browser-part3/)_ - -## Reference - -- https://developer.chrome.com/blog/inside-browser-part3/ -- https://blog.logrocket.com/how-browser-rendering-works-behind-scenes/ -- https://dev.to/arikaturika/series/17842 -- https://twitter.com/alexxubyte/status/1534201516063461376 diff --git a/topics/frontend/fundamental-end-to-end-frontend-testing-with-cypress.md b/topics/frontend/fundamental-end-to-end-frontend-testing-with-cypress.md deleted file mode 100644 index 16d576696..000000000 --- a/topics/frontend/fundamental-end-to-end-frontend-testing-with-cypress.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: Fundamental end to end frontend testing with Cypress -description: Learn how to perform end-to-end frontend testing with Cypress on a React autocomplete textbox, including setup, writing tests, and using Cypress with Mocha and Chai for assertions. -date: 2019-08-02 -tags: - - javascript - - testing -redirect: - - /0FP6Mw ---- - -![](assets/fundamental-end-to-end-frontend-testing-with-cypress_7578257d37750f65869976d63ebb2b57_md5.webp) - -## Introduction - -End-to-end testing is a methodology used to test whether the flow of an application is performing as designed from start to finish. It’s like testing as a user perspective and not involve thing like assert internal code state or mock… etc - -Cypress is an e2e testing framework that offers simplicity API and performant test runner - -## Application being tested - -![](assets/fundamental-end-to-end-frontend-testing-with-cypress_706c9f06c1ddbab02d7e04195d24bc16_md5.webp) - -The application that we are going to test against is autocomplete textbox. Features - -- Enter filter text and render list item that has text match filter text -- Select item in the list will replace filter text with text in the list item - -## Installation - -Clone this project for application setup (This repo. Install project dependencies with yarn install) - -Since this is a react project using custom webpack template. Here is some command to start the project: - -- yarn dev: start develop using webpack hot replacement server -- yarn build: build production application - -We must first start run yarn dev for testing server at localhost:9000 -To start writing test, create a npm script that starts cypress with npm command: cypress open - -```plain_text -"scripts": { - "e2e": "cypress open" -}, -``` - -Try running command yarn e2e first time, it will create folder named cypress. Create file named AutoComplete.spec.js in e2e/integration folder. Test files are located in `cypress/integration` by default, but can be configured to another directory. - -Cypress integrate with mocha test runner and chai assert library by default. Each describes statement create a section of the test case and each statement create a test case. Mocha also has a hook which will run the function inside it. My test file bellow use beforEach hooks which will run the function you pass in it in every test case - -```plain_text -describe('Auto Complete', function () { - beforeEach(function () { - cy.visit('localhost:9000') - }) - it('Filter source of the textbox correctly', function () { - - }) - it ('Set textbox input when click into list item', function () { - - }) -} -``` - -![](assets/fundamental-end-to-end-frontend-testing-with-cypress_dabaf075b757602a5af2c6bfcead3283_md5.webp) - -Run command yarn e2e it will show window and browser like this. A window is for select test and the browser is for testing result -E2E Testing -In the first test case: Filter source of the textbox correctly. That means we going to type in the textbox and expect it’s return list of item that includes the text in the textbox. Cypress syntax is chainable, you can chain it like - -`cy.get('input').type('abc')` - -The command above will get the element input tag and type word ‘abc’ in it. Cy.get take string parameter as CSS Selector. should statement allow you to do assertion. You must chain it after getting statement. - -The should command allow us to do assertion.Should have two kind syntaxes: - -- should(‘assertion_name, assertion_value) -- should(assertion_function) - -With type 1, you can view all available assertion_name at - -```javascript -it("Filter source of the textbox correctly", function () { - // get input and type 'one' in it - cy.get("input").type("one"); - - // as seed data: one should return 1 li element have text one - cy.get("li").should("have.length", 1); - - // li:first-of-type: select first element of li. This is psuedo css selector - cy.get("li:first-of-type").should("contain", "one"); -}); -``` - -With type 2, you pass a function in should that allow you to do multiple asserts. The function takes one parameter which is Jquery object. Since cypress builtin with chai assert library, you can use chai assertion to assert it. - -```javascript -it("Set textbox input when click into list item", function () { - cy.get("input").type("on"); - cy.get("li:first-of-type").click(); - - // Should assertion type 2 - cy.get("input").should(($input) => { - // $input is jquery wrapper. expect().to.be(xyz) is chai assertion - expect($input[0].value).to.be.equal("one"); - }); -}); -``` - -After inserting e2e test into the spec file. Run yarn e2e again to see the result of the test: - -![](assets/fundamental-end-to-end-frontend-testing-with-cypress_8fbb0902507f83afa2b0ef1bc5f830a0_md5.webp) - -## Conclusion - -In this article, I didn’t cover all feature of cypress, mocha task runner and chai assertion library. You can check its document at: - -- Chai assertion: [https://www.chaijs.com/api/bdd/](https://www.chaijs.com/api/bdd/) -- Cypress: [https://docs.cypress.io](https://docs.cypress.io/) - -You can view the full working repository at [https://github.com/PhmNgocNghia/TestingReact](https://github.com/PhmNgocNghia/TestingReact) diff --git a/topics/frontend/hsl-color.md b/topics/frontend/hsl-color.md deleted file mode 100644 index 335369b2b..000000000 --- a/topics/frontend/hsl-color.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: HSL color -description: HSL is the answer to resolving all of the painful points of Hexadecimal color codes. -date: 2022-09-02 -authors: - - trankhacvy -github_id: trankhacvy -tags: - - frontend - - hsl -redirect: - - /elx7QA ---- - -Frontend engineers use Hexadecimal color codes to represent colors, but they have some limitations: - -- Hexadecimal color codes are difficult to write and adjust, requiring the use of a third-party application to get right. -- Hexadecimal color codes can be difficult to remember and use, especially for designers. - -HSL is the answer to resolving all of the painful points. - -## What is HSL? - -**HSL** stands for **hue**, **saturation**, and **lightness**. It’s based on the RGB color wheel. Each color has an angle and a percentage value for the saturation and lightness values. - -- **Hue**: Think of a color wheel. Around 0o and 360o are reds. 120o is where greens are and 240o are blues. Use anything in between 0-360. Values above and below will be modulus 360. - - ![](assets/hsl-color_hls-color.webp) - -- **Saturation**: 0% is completely desaturated (grayscale). 100% is fully saturated (full color). - - ![](assets/hsl-color_hls-hue.webp) - -- **Lightness**: 0% is completely dark (black). 100% is completely light (white). 50% is average lightness. - - ![](assets/hsl-color_hls-lightest.webp) - -## Using HSL - -### Darker/lighter colors - -Imagine you're creating a button component and you want it to appear darker on hovering to increase its contrast. You can do this easily with the help of HSL. - -![](assets/hsl-color_using-hls.webp) - -```css -:root { - /* brand color: #E13F5E */ - --primary-h: 349; - --primary-s: 73%; - --primary-l: 56%; -} - -.button { - background-color: hsl(var(--primary-h), var(--primary-s), var(--primary-l)); -} - -.button:hover { - --primary-l: 40%; -} -``` - -### Color palette - -By altering the `lightness`, we can create a set of shades for a color that can be used throughout the UI where possible. - -![](assets/hsl-color_hls-color-pallete.webp) - -## HSL transparency (HSLa) - -It works exactly the same as with RGB, just add `alpha` channel with a value from 0 to 1. 0 is fully transparent. 1 is fully opaque. 0.5 is 50% transparent. - -```css -hsla(349, 73%, 56%, 0.5) -``` - -## References - -- https://css-tricks.com/hsl-hsla-is-great-for-programmatic-color-control/ -- https://www.smashingmagazine.com/2021/07/hsl-colors-css/ -- https://tsh.io/blog/why-should-you-use-hsl-color-representation-in-css/ diff --git a/topics/frontend/html-inert.md b/topics/frontend/html-inert.md deleted file mode 100644 index 9f9974c48..000000000 --- a/topics/frontend/html-inert.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: HTML inert -description: "`inert` is a boolean HTML property that makes the browser ignore the user input events for an element, including focus events and events from assistive technologies." -date: 2022-06-12 -authors: - - zlatanpham -github_id: zlatanpham -tags: - - a11y - - accessibility - - frontend - - html -redirect: - - /MM8itg ---- - -`inert` is a boolean HTML property that makes the browser ignore the user input events for an element, including focus events and events from assistive technologies. - -When an element has `inert` attribute, it will be removed from the tab order (interactivity), and from the accessibility tree (discoverability). While discoverability, previously, could be suppressed by `aria-hidden=true`, interactivity is more difficult as developers normally have to combine with `tabindex="-1"` to get it disabled. - -The born of `inert` helps to eliminate the need for [[Focus trap | focus trap]] which is being combined with `aria-hidden` to improve the accessibility of a web page containing an open dialog (see the attachment), though we still have to manually move the focus to the dialog and return focus the element that triggered it after the dialog is closed. - -![](assets/html-inert_pasted-image-20220612104313.webp) - -```html - - -
...
- - - -
...
-
...
-``` - -#### References - -- https://web.dev/articles/inert -- https://patrickhlauke.github.io/aria/demos/modal/index-aria-inert.html diff --git a/topics/frontend/i18n-frontend-guideline.md b/topics/frontend/i18n-frontend-guideline.md deleted file mode 100644 index de81d4e60..000000000 --- a/topics/frontend/i18n-frontend-guideline.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: I18n frontend guideline -description: Learn how to implement multi-language support with locale detection, internationalized routing, RTL/LTR text handling, and i18next formatting in React.js and Next.js front-end applications. -date: 2023-04-03 -tags: - - performance - - engineering - - frontend - - web -redirect: - - /COklGg ---- - -![](assets/i18n-frontend-guideline_0d5e0a2c5a795b96f65caa5b7a360578_md5.webp) - -In a front-end web application, locales are used to determine the language and geographic location of the user and to display the appropriate content to them. - -When a user visits a web page, the application will first detect their locale, then use this information to determine which version of the content to display. - -To support multiple languages and locales, the application needs to have the necessary content available for each locale. This can be achieved by creating separate files or resources for each locale or by using a localization library like `**i18next**` that allows for dynamic localization based on the user's detected locale. - -In this frontend guideline, we will discuss general principles and provide examples using `i18next` with `React.js` - -## Locale detection - -There are several ways to detect the user's locale (i.e., their language and geographic location) in a front-end application: - -- **Browser settings**: The user's browser settings may indicate their preferred language. This information can be accessed through the `navigator.language` or `navigator.userLanguage` property in JavaScript. -- **IP geolocation**: The user's IP address can be used to determine their geographic location. This can be done by making a request to a geolocation service, such as MaxMind or IP-API, which returns the user's location based on their IP address. -- **Server-side detection**: An HTTP header that relays these language preferences to the server with each request. This is the `Accept-Language` header, and it often looks something like this: `Accept-Language: en-CA,ar-EG;q=0.5`. -- **Use of cookies or web storage**: The user's locale preference can be stored in a cookie or HTML5 web storage when the user first interacts with the application. The locale would then be read from this location for all subsequent requests. - -There are a few libraries where that offer locale detection based on the settings listed above. For instance, In React.js, there is the `i18next-browser-languageDetector` library that detects language based on: - -- Cookie -- LocalStorage -- Navigator -- Query(`?lng=LANGUAGE`) -- HtmlTag -- Path -- Subdomain. - -Another example would be Next.js, where the locale will be automatically detected based on the `Accept-Language` header and the current domain. Locale detection is enabled by default. - -## Internationalized routing - -Internationalized routing is a way to handle different URLs for the same page based on the user's detected locale. There are two types of URL routing: - -- **Sub-path routing** (e.g. example.com/en/home, example.com/fr/home) -- **Domain routing** (e.g. example.en, example.fr) - -For example in React.js, the routing process can be implemented like this - -![](assets/i18n-frontend-guideline_9354a1ef08eeec42a93ec4329cf358c4_md5.webp) - -With Next.js, there is built-in support for internationalized routing since `v10.0.0` - -```javascript -// next.config.js -module.exports = { - i18n: { - locales: ["en-US", "fr", "nl-NL", "nl-BE"], - defaultLocale: "en-US", - domains: [ - { - // Note: subdomains must be included in the domain value to be matched - // e.g. www.example.com should be used if that is the expected hostname - domain: "example.com", - defaultLocale: "en-US", - }, - { - domain: "example.fr", - defaultLocale: "fr", - }, - { - domain: "example.nl", - defaultLocale: "nl-NL", - // specify other locales that should be redirected to this domain - locales: ["nl-BE"], - }, - ], - }, -}; -``` - -## Supports LTR and RTL text - -For some languages, such as Arabic, the letters are arranged from right to left. To ensure that your application supports Right-To-Left (**RTL**) layout rendering for such languages, you need to add **LTR** or **RTL** support. - -To add **LTR** or **RTL** support to the application, we will set the `dir` attribute on the `body` element dynamically in the `index.html` file. - -You can also set the `dir` attribute on global components such as `Header` and `Footer`. - -Here's an example of setting the `dir` attribute dynamically in the `App` component: - -```javascript -import React from "react"; -import { useTranslation } from "react-i18next"; -import "./App.css"; - -function App() { - const { t, i18n } = useTranslation(); - document.body.dir = i18n.dir(); // return ltr or rtl of current language - return
{t("welcome")}
; -} - -export default App; -``` - -Here's an example of setting the `**dir**` attribute on a global `**Header**` component: - -```javascript -import { useEffect } from 'react'; -import { useTranslation } from 'next-i18next'; - -const Header = () => { - const { t, i18n } = useTranslation('common'); - const localeDir = i18n.dir(); // return ltr or rtl of current language - - useEffect(() => { - document.querySelector('html')?.setAttribute('dir', localeDir); - }, [localeDir]); - - return ( -
- {...} -
- ); -}; - -export default Header; -``` - -## Formatting - -Starting from **i18next version 21.3.0**, you can take advantage of the built-in formatting functions based on the **[Intl API](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl)** for the following formats: - -- **Number** -- **Currency** -- **DateTime** -- **RelativeTime** -- **List** - -With these built-in formatting functions, you can easily format and localize various types of data in your application to match the user's preferred language and regional settings, improving the overall user experience of your application. - -Here's an example of how you can use them: - -```javascript -// Translation JSON -{ - "number": "Number: {{val, number}}", - "currency": "Currency: {{val, currency(USD)}}", - "dateTime": "Date/Time: {{val, datetime}}", - "relativeTime": "Relative Time: {{val, relativetime}}", - "list": "List: {{val, list}}", - "weekdays": [ - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday" - ] -} - -// Number -i18next.t('number', { val: 1000 }); // --> Number: 1,000 -i18next.t('number', { val: 1000.1, formatParams: { val: { minimumFractionDigits: 3 } } }); // --> Number: 1,000.100 - -// Currency -i18next.t('currency', { val: 2000 }); // --> Currency: $2,000.00 -i18next.t('currency', { - val: 2000.12, - currency: 'CAD', - locale: 'fr-CA' -}); // --> Currency: 2 000,12 $ CA - -// DateTime -i18next.t('dateTime', { val: new Date(Date.UTC(2012, 11, 20, 3, 0, 0)) }); // --> Date/Time: 12/20/2012 -i18next.t('dateTime', { - val: new Date(Date.UTC(2012, 11, 20, 3, 0, 0)), - formatParams: { - val: { weekday: 'long', year: 'numeric', month: 'long', day: 'numeric' }, - }, -}); // --> Date/Time: Thursday, December 20, 2012 - -// RelativeTime -i18next.t('relativeTime', { val: 3 }); // --> Relative Time: in 3 days - -// List -i18next.t('list', { - val: i18next.t('weekdays', { returnObjects: true }), -}); // --> List: Monday, Tuesday, Wednesday, Thursday, and Friday -``` - -## Conclusion - -Multilingual is a very important function for web or mobile applications nowadays, Users will come from all over the world and always ask for support for their language. Above is a guide to help you install multi-language for your application or website, You can implement it according to the instructions to automatically find the user's language, setup multi-language by domain or subpath, support LTR and RTL text, and finally format of number, currency, time, list… - -If you have a difficult problem that you would like us to help you on, please feel free to submit a challenge request here. - -**Come be with us** -We’d love to have you in our next chapter, by all means. - -- Discover what we do: [dwarves.foundation](http://dwarves.foundation/) -- Meet our team: [discord.gg/dfoundation](http://discord.gg/dfoundation) -- Join the squad: [careers.d.foundation](http://careers.d.foundation/) - -Follow our journey - -- Fanpage: [facebook.com/dwarvesf](http://facebook.com/dwarvesf) -- LinkedIn: [linkedin.com/company/dwarvesf](http://linkedin.com/company/dwarvesf) -- Substack: [https://memo.d.foundation/](https://memo.d.foundation/) diff --git a/topics/frontend/javascript-modules.md b/topics/frontend/javascript-modules.md deleted file mode 100644 index f9327286c..000000000 --- a/topics/frontend/javascript-modules.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -title: JavaScript modules -description: Modules have always been a part of JavaScript development and providing mechanisms for splitting JavaScript programs up into separate modules that can be imported when needed was the absolutely necessity for JavaScript developers. -date: 2022-06-27 -authors: - - namtrhg -github_id: namtrhg -tags: - - frontend - - javascript - - modules -redirect: - - /exuqdw ---- - -**Modules** have always been a part of JavaScript development and providing mechanisms for splitting JavaScript programs up into separate modules that can be imported when needed was the absolutely necessity for JavaScript developers. - -> "Modules are like building blocks that we can create software by combining." - -### So what is a module? - -A module is just simply a file that exports its own code. - -### Why do we need it? - -- Modules gives our code **structure** and **boundaries**. One's code will be hard to organize and maintain if they don't use modules and no one wants all the codes mingling with each others. -- Modules also allow developers share code to the world through a package manager like NPM. - -## History of JavaScript's modules - -In order to understand about the development of JavaScipt's modules, we need to walk through 4 stages: Vanilla, IIFEs, CommonJS and ESM. - -### Vanilla (1990s - early 2000s) - -When JavaScript was first invented by Brendan Eich in 1995, it was known to have been created in 10 days and the creator himself doesn't think that his programing language would be as popular as today. - -> “No one thought JavaScript would be used at the wide scale it is. Not just reaching lots of people on the Web, but large application like Gmail… To write large code, you don’t just want this little snippet language that I made for beginners…“ Bredan Eich (Creator of JavaScript) - -So modules aren't a thing back then, developers would mostly used inline-scripting and script-tags and no one would think of scalability, maintaining a large code base would be a nightmare because there is no structures and organizations. - -#### YAHOO manifesto - -Around 2006, a global manifesto was raised by a team of UI developers from YAHOO which was **"Global Variable are dangerous"**. The point they trying to make is that using global variable is a risky practice and all the people back then are engaging in. Global variables can spark conflict between naming if there are many people involving in developing the same program and since everyone can access it, it is not very secure. - -#### YAHOO Solution? - -Create a huge Global namespace object and parse all the variables in that object. - -> **Example**: YAHOO.value.getValue - -### IIFEs (early to late 2000s) - -IFFEs or Immediately invoked function expression was used with closures to keep the private data hidden, basically the ideal is that you only need to care of what the function returns and not what happen inside it. - -#### Example - -```js -var Dialogue = function () { - //private variables - var dialogue; - //exposed functions - return { - hello: function () { - dialog = "Hello Dwarves!"; - console.log(dialogue); - }, - goodbye: function () { - dialog = "Goodbye Dwarves!"; - console.log(dialogue); - }, - }; -}; -``` - -At this stage we have solve the Global namespace pollution problem but we haven't got to the point that we can swapping modules with one another and copy/paste function happens all the time. - -### CommonJS (early to mid 2010) - -Around 2009, people are hype about the potential of JavaScript, you can now run JavaScript on the server-side. And come with that a proposal was made for JavaScript to have a standard way to include other modules. Node created a module implementations name CommonJS. - -#### Implementation of CommonJS: - -- **Import**: using `require()` method. -- **Export**: **module** object using `module.exports`. - -This was design for server development. - -#### Synchronus modules loading - -- **MODULE.\_LOAD**: Node caches it's modules (Node check for cache files, if it not there it will create an instance and caches it). -- **MODULE.\_COMPILE**: Create a `require` function that speciffic for each modules and generate a wrapper function to scope variables the run the wrapper function. - -#### Node module wrapper function - -```js -(function (exports, require, module, __filename, __dirname) { - // Module code actually lives in here -}); -``` - -#### Example - -```js -//dialogue.js -var getDialog (function (){ - console.log("Hello Dwarves!"); -}) -module.exports = getDialogue; -``` - -```js -//main.js -const sayHello = require("./dialogue.js"); -``` - -At this time, CommonJs is a real noviation, organizing files and maintaining code has become much easier. - -### ESM (2010s - present) - -ESM or ES modules was supported by all major browsers and was the first **built-in module** in JavaScript and introduce top-level `await` functionality for module imports. - -#### Top-level await - -A feature accessible within modules is top level await. This implies that the await keyword is functional. It enables modules to function as sizable asynchronous functions, allowing code to be assessed before usage in parent modules without preventing the loading of siblings. - -#### ESM Asynchronus ? - -The loading and parsing for ESM modules is indeed asynchronous, but the execution of the code in them is synchronous and serialized based on the order they are imported. - -#### Implementation ESM: - -- **Import**: using `import` method. -- **Export**: using `export` method. - -#### Example - -```js -//Export syntax - -//dialog.js -export default const dialog = "Hello Dwarves!"; - -//utils.js -export const getHours = () => //do something; -export const getMinutes = () => //do something; -``` - -```js -//Import syntax - -import dialog from "./dialog.js"; - -import * as utility from "./utils.js"; -import { getHours, getMinutes } from "./utils.js"; -``` - -#### ESM in the browsers - -ESM has a specfic module type in the browser which they are processed differently and create a fallback for older browsers using bundle. - -```html - - - - -``` - -### CommonJS vs ESM - -- Right now ESM was supported on all browsers and the latest version of Node except version below version 12. -- ESM waits to execute any code in a module until all of it's imports have been loaded and parsed, then does the binding/side-effects stuff in the relative order that they happen. -- ESM imports are asynchronous (which also allows for top-level `await`). -- CJS executes imports as it finds them, blocking until they finish. -- CJS works in Node but does **not** work in browsers. - -### Conclusion - -Although CommonJS can still be used for server development since synchronous wasn't an issue and it was built by Node, using ESM is still the better way to keep the syntax consistent if you were to develop client-side and server-side with JavaScript. - -#### Reference - -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules -- https://dev.to/adamcoster/commonjs-and-esm-importexport-compatibility-by-simple-example-50pl -- https://www.w3schools.com/js/js_modules.asp -- https://nodejs.org/api/modules.html -- https://www.infoworld.com/article/2653798/javascript-creator-ponders-past--future.html diff --git a/topics/frontend/micro-frontends-microservices-for-frontend-development.md b/topics/frontend/micro-frontends-microservices-for-frontend-development.md deleted file mode 100644 index dca3dc694..000000000 --- a/topics/frontend/micro-frontends-microservices-for-frontend-development.md +++ /dev/null @@ -1,243 +0,0 @@ ---- -title: Micro frontends microservices for frontend development -description: Micro Frontends is an architectural style where independently deliverable frontend applications are composed into a greater whole. -date: 2021-07-09 -authors: - - zlatanpham -tags: - - micro-frontend -redirect: - - /DZMsRw ---- - -## What is Micro-frontend ? - -> An architectural style where independently deliverable frontend applications are composed into a greater whole - -## Benefits - -- Smaller, more cohesive, and maintainable codebases -- More scalable organizations with decoupled, autonomous teams -- The ability to upgrade, update, or even rewrite parts of the frontend in a more incremental fashion than was previously possible -- Easier maintenance: Keeping frontend repositories small and specialized allows them to be more easily understood, and this simplifies long-term maintenance and testing. - -## Micro frontends in actions - -### Build-time integration - -- Publish each micro frontend as a package and have the container application include them all as library dependencies. - -```javascript -{ - - "name": "@shop/container", - - "version": "1.0.0", - - "description": “Ecommerce website", - - "dependencies": { - - "@shop/products": "^1.0.0", - - "@shop/order": "^1.0.0", - - "@shop/user-profile": "^1.0.0" - - } - -} -``` - -### Cons - -- Have to re-compile and release every single micro frontend in order to release a change to any individual part of the product. - -## Run-time integration via iframes - -- Use `iframe` to connect other micro frontends - -`````javascript - - - - - Shop - - - - - -

Welcome to Shop

- - - - - - - - - ```` - - -### Cons - -- Difficult to make the page responsive -- Difficult to integrate between different parts of an application, make routing, history, and deep-linking more complicated - -## Run-time integration via JavaScript - -- Each micro frontend is included on the page using a ` - - - - - -
- - - - - - -````` - -## Run-time integration via [Web Components](https://developer.mozilla.org/en-US/docs/Web/Web_Components) - -- Define each micro frontend as an HTML custom element for the container to instantiate. - -```javascript - - - - - Shop - - - - - -

Welcome to Shop

- - - - - - - - - - - -
- - - - - - -``` - -### Cons - -- Don't support server-side rendering -- No 100% [browser support](https://caniuse.com/#search=custom%20elements) - -## Cross-application communication - -1. [Custom event](https://developer.mozilla.org/en-US/docs/Web/Events/Creating_and_triggering_events) -1. Cross-application communication via routing - -## Serverside rendering / universal rendering - -- Server-side rendering is always a tricky problem. -- Use [Server Side Includes](https://en.wikipedia.org/wiki/Server_Side_Includes) to plug in page-specific content from fragment HTML files - -# Conclusion - -- Duplication of dependencies => increase payload size -- App performance -- Learning curve -- Only suitable for medium, large projects - -## References - -- [Micro frontends](https://micro-frontends.org/) -- [Micro frontends from Martinfowler](https://martinfowler.com/articles/micro-frontends.html) -- [Micro frontends—a microservice approach to front-end web development](https://www.tomsoderlund.com/programming/micro-frontends-a-microservice-approach-to-front-end-web-development) -- [Awesome micro frontends](https://github.com/ChristianUlbrich/awesome-microfrontends) diff --git a/topics/frontend/mixpanel.md b/topics/frontend/mixpanel.md deleted file mode 100644 index 3e2de7a6d..000000000 --- a/topics/frontend/mixpanel.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Mixpanel -description: Mixpanel is a powerful analytics platform that assists companies in monitoring and analyzing user behavior on their websites and mobile applications. -date: 2023-02-13 -authors: - - namtrhg -github_id: namtrhg -tags: - - analytics-platform - - analytics-tools - - frontend - - typescript -redirect: - - /a0FOXA ---- - -Mixpanel is a powerful analytics platform that assists companies in monitoring and analyzing user behavior on their websites and mobile applications. With Mixpanel, you can analyze user activity, evaluate the performance of your products, and improve user experience through data-driven decisions. ![](assets/mixpanel_dashboard.webp) - -## Key features - -Some key features of Mixpanel include: - -1. **Real-time analytics**: Mixpanel offers data on user activity in real-time, enabling organizations to instantly see patterns and take wise decisions. -2. **Custom event tracking**: Businesses can better understand how customers engage with their products by tracking custom events like button clicks, form submissions, and in-app payments using Mixpanel. -3. **Funnels and retention**: Mixpanel offers robust tools for funnel analysis to assist companies better understand how customers interact with their products and how to increase user retention. -4. **People analytics**: Mixpanel allows businesses to segment and understand their user base by tracking user behavior, demographics, and other key metrics. -5. **Mobile analytics**: Mixpanel offers robust mobile **analytics** **that** enable businesses to track user behavior on both iOS and Android platforms. -6. **Segmentation**: Mixpanel enables businesses to create custom segments of users, based on specific behaviors or attributes, to understand how different groups of users engage with their products. -7. **Integration**: Mixpanel integrates with a wide range of tools, including popular CRM and marketing automation platforms, making it easy to use alongside other technologies. - -## Integration with React - -1. Create a Mixpanel account and get a Mixpanel project token. To use Mixpanel, you need to sign up for an account and create a project. When you create a project, you receive a unique token that is used to identify your project. -2. Install the Mixpanel library: You can install the Mixpanel library by running the following command in your terminal: - -```zsh -yarn add mixpanel-browser -``` - -3. Import the Mixpanel library into your React component: In your React component, import the Mixpanel library as follows: - -```ts -import mixpanel from "mixpanel-browser"; -``` - -4. Initialize Mixpanel: In your component's constructor or useEffect hook, initialize Mixpanel with your project token: - -```ts -//index.tsx -mixpanel.init("YOUR_MIXPANEL_PROJECT_TOKEN"); -``` - -5. Track events: To track events in your React application, you can use the Mixpanel `track` method. For example: - -```ts -mixpanel.track("Button clicked"); -``` - -6. Use Mixpanel's identify method to track unique users: The Mixpanel `identify` method allows you to track unique users in your application. You can call this method with a unique identifier for the user, such as an email address or user ID: - -```ts -mixpanel.identify("user_id"); -``` - -7. Additionally you could use people.set.method to create a profile for the users - -```ts -mixpanel.people.set({ - $name: profile?.fullName, - $email: profile?.email, - $company_name: profile?.company?.name, - $company_id: profile?.company?.id, - $user_id: profile?.id, -}); -``` - -As you can see in the images bellow all users interact with the app will be listed. ![](assets/mixpanel_user_profiles.webp) - -You can observe all the events created by the users. ![](assets/mixpanel_user_activites.webp) - -## References - -- -- diff --git a/topics/frontend/mpa-spa-and-partial-hydration.md b/topics/frontend/mpa-spa-and-partial-hydration.md deleted file mode 100644 index 7cb53bc6f..000000000 --- a/topics/frontend/mpa-spa-and-partial-hydration.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: MPA, SPA and partial hydration -description: MPA, SPA and partial hydration are three different approaches to building web applications. -date: 2022-03-24 -authors: - - huygn -github_id: huygn -tags: - - frontend -redirect: - - /nBMWig ---- - -If you've been on Twitter lately, you might've seen the term "Partial Hydration" as well as MPA going along and hyped by web advocates, wondering what about them? - -## MPA - -"Multi-page-app", opposes the more commonly known "Single-page-app". The idea is now that we're back to the web era of 10 years ago of truly "static" websites. Each link navigation triggers a full-page reload to the new HTML page with no SPA-style navigation (a navigation style that doesn't trigger a full page reload, but replace the current page with the new page, similar to an app-like experience). - -### Why? - -The web community has spent huge efforts to make SPA faster and more SEO friendly than how it was originally introduced: SSR and streaming mechanisms allow for making meaningful content visible as fast as possible. But we've come to a realization: nothing beats an MPA in terms of speed of page loading and SEO. - -SSR did a wonderful job bringing faster page loads to SPA, but the problem still exists: we've been sending really huge, mostly unused, JavaScript payloads to users. - -## Why not SPA? - -With the current advanced web technologies, mainly web libraries/frameworks like React, Preact, Vue, Svelte, etc - most companies opt to build their web products completely with those libraries. Since these frameworks are excellent in building SPAs, we've been seeing a lot more SaaS products as SPA now more than ever. - -SPA isn't technically a bad thing, it makes navigation feels more "native" and smooth - with features like preloading and background fetching for new pages that the user might likely visit. In fact, it's so great that a lot of recognizable web software are built as an SPA: Facebook, Twitter, Notion, etc... - -However, SPA does come with a cost that is not so obvious at first glance: **a large initial bundle size**. - -### The SPA story - -The way most SPAs works is that (almost) everything in our web UI is composed with JavaScript (with or without frameworks). JavaScript is run this way to create HTML at run-time, with the most basic form of this pattern being `document.createElement("div")`. Now, imagine if every text, button, section, popup, and relevant UI components of our website were built using this pattern. Why is it not-so-good to use SPA? - -The answer: [It's the cost of JavaScript](https://timkadlec.com/remembers/2020-04-21-the-cost-of-javascript-frameworks/). Unlike raw HTML, using JavaScript to create HTML implies unseen overhead. This amounts to the cost of downloading, parsing, and executing JavaScript. Even before we can see anything on the screen, the browser has to run all of this to completion in order to show us something meaningful. Depending on the context, this effect can cause long delays between when a user visits a site and when a meaningful content is visible to that user. Nowadays, web performance tools measures and categorizes this delay as one of the most [important metric for frontends: (LCP)](https://web.dev/lcp/). - -### SSR & hydration - -Then we have SSR "Server Side Rendering" - used to improve LCP and provide better SEO capability for SPAs. In React, this feature is handled with `ReactDOMServer.renderToString`. The idea is simple: render the whole React tree to raw HTML on server and return it to browser. The user will then be able to see content immediately after HTML is downloaded. - -Now we have the best of both worlds, with SPA & SSR - an app-like experience with fast content delivery time! - -Not so fast. We’re still missing one final piece - the raw HTML returned by our server won't be fully interactive (e.g: the "counter" state won't change when we click buttons). In order to make our page interactive, we have to "hydrate" it. - -**Hydration** is the process of turning the raw HTML we returned earlier to a fully interactive React tree of components. For React, this can be done using the `ReactDOM.hydrate` method. So full process would be: - -1. `renderToString` & send the (html) string to clients on the server -2. Loads the corresponding React component and hydrate it on the client: - - `import Page from "pages/home"` - - `hydrate(Page, document.getElementById("app"))` - -"Sooo that’s still all good!". Yes, except for one thing: do we really need ALL our components to be interactive? Probably not, especially for cases when we're building a basic landing/marketing site using React, where most of our content is static except for subscribe forms, sliders, etc. Hydrating the whole page in this scenario is called _hydration waste_: only some components are actual interactive components, yet we hydrate everything from the top down, which in turn makes our [time-to-interactive (TTI)](https://web.dev/interactive/) longer. - -## Partial hydration - -This concept proposes hydrating only parts of the entire site - parts where we need interactivity, this in turn help us to ship less JavaScript to the client by only hydrating demanding components. Thus, we improving page load time & time-to-interactive. As of now, only some static site frameworks support this out of the box: [Astro](https://docs.astro.build/core-concepts/component-hydration) & [Marko](https://markojs.com/). - -To implement this from scratch in a React project, although doable, requires [quite a lot of implementation](https://medium.com/@luke_schmuke/how-we-achieved-the-best-web-performance-with-partial-hydration-20fab9c808d5), and it may be hard to debug issues in the process. - -## MPA, SPA or Partial Hydration? - -Same old answer: it depends. If you’re building an analytics dashboard, sports betting portal, or just a better version of PowerPoint, stick with SPA (and SSR, if needed). These types of websites have lots of interactive UI to make Partial Hydration just not worth it. - -Otherwise, if it's your landing site, similar to [netlify.com](http://netlify.com/) or [dwarves.foundation](https://dwarves.foundation/), which essentially just has only 1-2 pieces of UI that require JavaScript to run, and you’re building it with React: try to apply Partial Hydration where possible. While `react-static` or `gatsby` does not (yet) support partial hydration, you can try out [astro.build](https://astro.build/). Here, partial hydration is a first class citizen, and it comes with a decent static site generator with good React support. - -For MPA, if you prefer to keep building your site with React, you can still make it with an MPA. Just do SSR on the server and AVOID hydrating the whole page on the client. Obviously interactive components won't be interactive, but at least it's a good start. diff --git a/topics/frontend/parallelism-in-javascript.md b/topics/frontend/parallelism-in-javascript.md deleted file mode 100644 index 72519e07c..000000000 --- a/topics/frontend/parallelism-in-javascript.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Parallelism in JavaScript -description: Parallelism in JavaScript is a technique to improve the performance of web applications by executing multiple tasks simultaneously. -date: 2022-04-04 -authors: - - zlatanpham -github_id: zlatanpham -tags: - - parallelism - - concurrency - - frontend - - javascript -redirect: - - /T7Gjkg ---- - -Most of applications nowadays are created with the aim to be on the web. JavaScript has become a mainstream programming language to support the direction. However, being designed as a single-threaded language doesn't give JavaScript the power to support heavyweight applications as when we talk about image/video processing, online games or even data mining. This naturally creates a demand to support Parallelism in the language. - -### Web worker - -Web worker is one of the initial attempts that successfully brings the concept of parallelism to browsers. A web worker creates a JavaScript thread performing tasks, without interfering with the user interface. - -Web workers run in a different context from the main thread. Because of that, `document` and `window` objects of the main thread are not accessible in web workers. However, it is possible to use XMLHttpRequest, WebSocket or data storage mechanism. - -There are two kinds of workers: shared and dedicated. A dedicated worker can be only accessed from the script that initiated it while the state of shared workers can be accessed from multiple scripts. - -Communication between the worker thread and the main thread is via a system of messages where both sides send data using the `postMessage()` method, and respond to messages via the `onmessage` even handle. The data is serializable. It means data is copied rather than shared. So, there is a performance bottleneck if the amount of transferred data is huge. - -```javascript -# main.js -const myWorker = new Worker('./worker.js'); - -myWorker.postMessage(1); - -myWorker.onmessage = function(event){ - console.log(`Receieved ${event.data} from web worker`) -} -``` - -```javascript -# worker.js -onmessage = function(event){ - console.log(`${event.data} received from main script`); - - if(event.data === 1){ - postMessage(2); - } -} -``` - -### WebCL and other OpenCL based frameworks - -Another approach to apply the concurrency model for web applications is [WebCL](https://www.khronos.org/webcl). It is a JavaScript binding to the OpenCL standard that enables web applications to harness GPU and multi-core CPU parallel processing from within a web browser, thus, enabling the possibility to deploy computationally intensive applications. Currently, no browsers natively support WebCL but non-native add-on can be used to embed WebCL in web browsers. - -[RiverTrail](https://github.com/IntelLabs/RiverTrail/) is another framework that relies on OpenCL to provide support for data parallel programming in JavaScript. It works by extending JavaScript with new data-parallel constructs that are translated at runtime into a low-level hardware abstract layer. - -However, most of OpenCL based frameworks developed during the period are no longer being in active development as no browser vendors have a concrete plan to implement it in their environment at the moment. It seems the attention to boost the performance of the web is bet on different approaches such as [WebAssembly](https://developer.mozilla.org/en-US/docs/WebAssembly) or [WebGPU](https://www.w3.org/TR/webgpu/). So, at the time of writing, Web worker is the only approach to bring parallelism in web applications in production. - -#### References - -- https://www.khronos.org/registry/webcl/specs/latest/1.0/#3.1 -- https://en.wikipedia.org/wiki/WebCL -- https://intellabs.github.io/RiverTrail/ diff --git a/topics/frontend/parse-don-t-validate-in-typescript.md b/topics/frontend/parse-don-t-validate-in-typescript.md deleted file mode 100644 index bbf20a15f..000000000 --- a/topics/frontend/parse-don-t-validate-in-typescript.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -title: "Parse, don't validate in TypeScript" -description: 'The "parse, don''t validate" approach is about processing incoming data and failing in a controlled manner if parsing fails.' -date: 2023-01-05 -authors: - - namtrhg -github_id: namtrhg -tags: - - frontend - - typescript - - validation -redirect: - - /N4d1mg ---- - -The _**"parse, don't validate"**_ approach is all about processing incoming data and failing in a controlled manner if parsing fails. It is all about leveraging trustworthy, secure, and typed data structures within your code and ensuring that all incoming data is handled at the very edges of your systems. Instead of passing receiving data deep into your code, parse it immediately and fail quickly if necessary. - -Parsing is better than validation because it requires you to explicitly handle every incoming data. It provides a type-safe method of working and makes it difficult to spread harmful material throughout your apps and data storage. However, it is true that parsing frequently incorporates data validation. - -## Overview - -### What is parsing? - -- **Parsing** is the process of analyzing a string or symbol either in natural language, computer languages or data structures, conforming to the rules of a formal grammar. -- The **"_process of analyzing_"** and **"_conforming to the rules of a_ [_thingy_]"** are crucial here. _Thingy_ is our schema and type, which in this instance may be thought of as forming our _formal grammar_ (don't worry if you don't know what it means). _Process of analyzing_ is the work our code does when trying to fit data to the schema & type. The reason why we are saying “schema & type” is that we want them somehow to be the same thing, instead of two separate things that may or may not be in sync. - -### Examples in TypeScript with yup - -#### Example - -```ts -import * as yup from "yup"; - -let schema = yup.object().shape({ - name: yup.string().required(), - age: yup.number().required().positive().integer(), - email: yup.string().email(), -}); -// data from an API, from user, etc. Note that the type would be `any` -// (or perhaps `unkown`) as we don't really know what it looks like -// if it comes from outside -const data: any = { - name: "jimmy", - age: 24, -}; -// check validity -schema.isValid(data).then(function (valid) { - console.log("isValid?", valid); // => true - // do something with the data, however, it's still `any`/`unkown`.... -}); -``` - -We still don't have a _type_ for our 'data,' as you can see. It remains 'any/unknown'. Sure, we can typecast it, but it introduces a problem: we now have to maintain a'schema' and a 'type' separately, by hand, with nothing ensuring they match. - -#### Example with Typecast - -```ts -let userSchema = yup.object().shape({ - name: yup.string().required(), - age: yup.number().required().positive().integer(), - email: yup.string().email(), -}); - -type UserType = { - name: string; - age: number; - email: string; -}; -// check validity -schema.isValid(data).then(function (valid) { - const user = data as UserType; // it's valid so let's cast! -}); -``` - -Still, how can you ensure that 'userSchema' and 'UserType' are in sync? Do they even stand for the same thing? - -#### Example with parsing function - -```ts -// Let's use some custom type aliases for readability -type PositiveInteger = number; -type Email = string; -type URL = string; -// Type guards to validate invidiual values, fields -const isPositiveInteger = (x: any): x is PositiveInteger => - yup.number().required().positive().integer().isType(x); -const isEmail = (x: any): x is Email => - yup.string().required().email().isType(x); -const isString = (x: any): x is string => yup.string().required().isType(x); -// UserType again, now with our custom type aliases -type UserType = { - name: string; - age: PositiveInteger; - email?: Email; -}; -/** -parse, don't validate -compiler can help us quite a bit here to make sure the parsing -is actually correct -*/ -const parseToUserType = (x: any): UserType | Error => { - let { name, age, email, website, createdOn } = x; - if (!isString(name)) return new Error("invalid name"); - if (!isPositiveInteger(age)) return new Error("invalid age"); - email = isEmail(email) ? email : undefined; // optional, silently drop invalid values - return { age, name, createdOn, email, website }; -}; -// Business logic is pretty awesome now! -function myHandler(): Response { - const userType: UserType | Error = parseToUserType(data); - - if (userType instanceof Error) return { error: "Ohh there was a 400 error" }; - // use UserType normally, do what ever you want - return { message: `Welcome, ${userType.name}` }; -} -``` - -- Business logic can now parse any data to `UserType`. \_Just remember to check whether there’s an Error or not. -- We are using more precise types than `string` or `number`\_ due to how type aliases work. -- The error-prone, dangerous, not-that-well-type-checked code is limited to type guards. -- TS compiler makes sure our parser actually works. _We can safely write our_`_(x: any) => UserType | Error` parser function with good support from the type checker. - -### Reference - -https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/ https://en.wikipedia.org/wiki/Parsing https://itnext.io/parse-dont-validate-incoming-data-in-typescript-d6d5bfb092c8 diff --git a/topics/frontend/prevent-layout-thrashing.md b/topics/frontend/prevent-layout-thrashing.md deleted file mode 100644 index 87daf1d93..000000000 --- a/topics/frontend/prevent-layout-thrashing.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -title: Prevent layout thrashing -description: Layout Thrashing happens, when you request layout information of an element or the document, while the layout is in an invalidated state. -date: 2022-09-11 -authors: - - thanhlmm -github_id: thanhlmm -tags: - - performance - - frontend -redirect: - - /cPgVWA ---- - -## What is layout thrashing - -![](assets/prevent-layout-thrashing_layout-thrashing.webp) - -Layout thrashing means forcing the browser to calculate a layout that is never rendered to the screen, which hurts performance. - -## Why - -Layout Thrashing happens, when you request layout information of an element or the document, while the layout is in an **invalidated state**. - -```js -// any DOM or CSSOM change flags the layout as invalid -document.body.classList.add("foo"); - -// reads layout == forces layout calculation -const box = element.getBoundingClientRect(); - -// write/mutate -document.body.appendChild(someBox); - -//read/measure -const color = getComputedStyle(someOtherBox).color; -``` - -Mixing Layout Read & Layout Mutation must wait for the browser to recalculate the layout and reflow to return your Layout value. - -![](assets/prevent-layout-thrashing_dont-touch-me.webp) - -## How to fix - -We can resolve the problem by isolating your reads from your writes. The steps would be: - -- Batch Read Layout first -- Then Mutate Layout later - -```js -// reads layout -const box = element.getBoundingClientRect(); -const color = getComputedStyle(someOtherBox).color; - -// write -document.body.classList.add("foo"); -document.body.appendChild(someBox); -``` - -Or use a library such as [fastdom](https://github.com/wilsonpage/fastdom) which abstracts those steps: - -```js -import fastdom from "fastdom"; - -function resizeAllParagraphsToMatchBoxWidth(paragraphs, box) { - fastdom.measure(() => { - const width = box.offsetWidth; - - fastdom.mutate(() => { - for (let i = 0; i < paragraphs.length; i++) { - paragraphs[i].style.width = width + "px"; - } - }); - }); -} -``` - -## How to debug - -Open `Performance Tab` on Dev tool, slow down your CPU, and click `Start Profiling`. - -![](assets/prevent-layout-thrashing_layout-thrashing-debug.webp) - -Find purple tasks and get info in detail: - - - -## List of commands causing layout thrashing we need to be careful when using it - -Generally, all APIs that synchronously provide layout metrics will trigger forced reflow/layout. Check out [this gist](https://gist.github.com/paulirish/5d52fb081b3570c81e3a) for additional cases and details. - -## Reference - -- https://gist.github.com/paulirish/5d52fb081b3570c81e3a -- https://afarkas.github.io/layout-thrashing/#/ diff --git a/topics/frontend/pure-css-parallax.md b/topics/frontend/pure-css-parallax.md deleted file mode 100644 index 53c6c3391..000000000 --- a/topics/frontend/pure-css-parallax.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -title: Pure CSS parallax -description: This article demonstrates how to use CSS transforms, perspective and some scaling trickery to create a pure CSS parallax scrolling website. -date: 2022-09-10 -authors: - - ngolapnguyen -github_id: ngolapnguyen -tags: - - animation - - css - - frontend - - html -redirect: - - /PqGoGA ---- - -This article demonstrates how to use CSS transforms, perspective and some scaling trickery to create a pure CSS parallax scrolling website. - -## Advantages of using pure CSS over JS - -Although using Javascript will give us more flexibility on how we want to construct our parallax effect, it also comes with the cost of performance & implementation complexity. We listen to the `scroll` event & modify the DOM with the handler, triggering needless reflows and paints. - -For more simple use cases, with pure CSS, we can: - -- Avoid messing with the browser's rendering pipeline -- Allow browsers to leverage hardware acceleration while rendering, ensuring consistent frame rates & a smooth scrolling experience -- Combine with other CSS features (e.g. responsive) - -## How it works - -First, let's establish some barebones markup: - -```html -
-
...
-
...
- ... -
-``` - -And the basic styles: - -```css -.parallax { - perspective: 1px; - height: 100vh; - overflow-x: hidden; - overflow-y: auto; -} - -.layer { - position: absolute; - top: 0; - right: 0; - bottom: 0; - left: 0; -} - -.layer-1 { - transform: translateZ(0); -} - -.layer-2 { - transform: translateZ(-1px); -} -``` - -The `parallax` class is where the parallax magic happens: - -- Defining the `height` and `perspective` style properties of an element will lock the perspective to its centre, creating a fixed origin 3D viewport. -- Setting `overflow-y: auto` will allow the content inside the element to scroll in the usual way, but now descendant elements will be rendered relative to the fixed perspective. This is the key to creating the parallax effect. - -The `layer` class defines a layer of content to which the parallax effect will be applied. The `absolute` position is optional, for the sake of display. You'll see what I meant in a moment. - -Finally, the `layer-{{n}}` class is used to set Z offset of the layers. If we consider the `parallax` container is a camera viewport, the Z offset will determine whether a layer is farther away, or closer to the viewport. **The farther away a layer is, the slower it'll appear to be scrolling.** - -Check it out in the CodePen below: - - - -## Common practices - -### Parallax with multiple sections - -Most parallax sites break the page into distinct sections where different effects can be applied. Here's how to do that. - -First, we need a `group` element to group our layers together: - -```html -
-
-
Layer 1.1
-
Layer 1.2
-
Layer 1.3
-
- ... -
-``` - -And now the styles: - -```css -.group { - ... - transform-style: preserve-3d; -} -``` - -The property `transform-style: preserve-3d` prevents the browser from flattening the `layer` elements, indicating that children of the element should be positioned in the 3D-space. More on the property [here](https://developer.mozilla.org/en-US/docs/Web/CSS/transform-style). - -One important rule to keep in mind when grouping elements is, **we cannot clip the content of a group**. Setting `overflow: hidden` on a `group` will break the parallax effect. Unclipped content will result in descendant elements overflowing, so we need to be creative with the `z-index` values of the groups to ensure content is correctly revealed/hidden as the visitor scrolls through the document. - -### Depth correction - -True to 3D transforms, elements that are farther away from the viewport will appear smaller than those that are closer. If we want them to appear to be rendered as their original size (e.g. same font-size and all), we can use the `scale` transform to do that: - -```css -.layer-2 { - transform: translateZ(-1px) scale(2); -} -``` - -The scale factor can be calculated with the following formula: - -``` -scale = 1 + (translateZ * -1) / perspective -``` - -### Debugging - -When you are working with parallax, it can be easier to get lost among the different layers. Taking a different perspective will allow you to know where everything is in the 3D space - which you can do by applying simple transform to the group elements: - -```css -.group { - transform: translate3d(700px, 0, -800px) rotateY(30deg); -} -``` - -You can try out all the common practices I have mentioned in the CodePen below: - - - -## References - -- https://keithclark.co.uk/articles/pure-css-parallax-websites/ -- https://developer.mozilla.org/en-US/docs/Web/CSS/transform-style -- https://www.youtube.com/watch?v=1wfeqDyMUx4 diff --git a/topics/frontend/recursively-export-file-pattern-in-javascript-es6-application.md b/topics/frontend/recursively-export-file-pattern-in-javascript-es6-application.md deleted file mode 100644 index 8d412207d..000000000 --- a/topics/frontend/recursively-export-file-pattern-in-javascript-es6-application.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: Recursively export file pattern in Javascript ES6 application -description: Learn how to simplify JavaScript ES6 imports by using recursive re-export patterns and automate index file creation with the @autogen-export package for cleaner component management. -date: 2019-09-07 -tags: - - javascript -redirect: - - /2fjm8w ---- - -![](assets/recursively-export-file-pattern-in-javascript-es6-application_08f14b555ce54599844167b5700622ca_md5.webp) - -## Introduction - -Imagine you have a lot of components and you imported it like this: - -![](assets/recursively-export-file-pattern-in-javascript-es6-application_c733b9fb01a2eb50f4a8895d2cd68acd_md5.webp) - -Instead of importing component like above. You would want to import your components like this: - -![](assets/recursively-export-file-pattern-in-javascript-es6-application_024be7746f1d8a3f25ffad7888a47caf_md5.webp) - -**The pros of this style are** - -- Your project structure now grouped into multiple namespaces. You can image It’s like C# namespace. Each namespace is a root folder in our project. -- Don’t have to remember exactly path of your components. - -**The cons are** - -- Can’t export members who name has been exported already. Eg: Component/A export A member so Component/B cannot export A member. - -## Implementation - -To implement this style, We use re-export statement which is all feature of Javascript ES6: [https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/export](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/export) - -It’s syntax like bellow. All named export or all export data from the module, whose path is something, will re-export from the file that use re-export syntax - -```javascript -export {named export} from 'somthing' -export * from 'something' -``` - -Then when you import the name using re-export syntax. It’s will contain member which It’s re-export. - -I implemented a small example at codesandbox: [https://codesandbox.io/s/admiring-hill-esw1j](https://codesandbox.io/s/admiring-hill-esw1j) - -In this example, I have a folder named component in the root folder. In the folder, I have component1 and component2 and a folder named `nested-component`. - -Inside that folder also have `nested-component1` and `nested-component2`. - -Let’s say from the app component in the directory I want to import component1,2; nested-component1,2. I have to import it like this - -```javascript -import React from "react"; -import ReactDOM from "react-dom"; -import Component1 from "../src/components/component1"; -import Component2 from "../src/components/component2"; -import NestedComponent1 from "../src/components/nested-component/nested-component1"; -import NestedComponent2 from "../src/components/nested-component/nested-component2"; -import "./styles.css"; -function App() { - return ( -
- - - - -
- ); -} -const rootElement = document.getElementById("root"); -ReactDOM.render(, rootElement); -``` - -I will refactor it by creating a file named index.js in a components folder that exports all content of Its child. - -```javascript -export { component1 } from "./component1"; -export { component2 } from "./component2"; -``` - -or - -```javascript -export { component1 } from "./component1"; -export { component2 } from "./component2"; -export { NestedComponent1 } from "./nested-component/nested-component1"; -export { NestedComponent2 } from "./nested-component/nested-component2"; -``` - -Please note that the export all identifier above is not re-export the export default state of the module. eg: In module A exported default a and exported named b. Only b will be re-exported. - -Then in your app component, you can - -```javascript -import { - component1 as Component1, - component2 as Component2, - NestedComponent1, - Nested, -} from "./components"; -``` - -## @Autogen-export Package - -I implemented those packages to automation the create export file job. It’s work with any ES6 Code (React, Vue) as long as you provide a correct babel configuration file for @babel/core parsed the code properly. -Auto-generate-export file is a utility tool which generates index file exported all of Its child folder (that contain exportable index file) and child file. - -It’s work with Typescript, Javascript ES6. It has many configurable options that allow you to tweak all of Its aspects: - -- Extension type of a generated file. -- Extension of a file that will be parsed. -- Ignore specific folder. - -It has been published as an NPM package. - -- [https://www.npmjs.com/package/@autogen-export/core](https://www.npmjs.com/package/@autogen-export/core) -- [https://www.npmjs.com/package/@autogen-export/cli](https://www.npmjs.com/package/@autogen-export/cli) - -I also created examples on how to use those packages: [https://github.com/phmngocnghia/AutoGenerateReExportFile/tree/master/examples](https://github.com/phmngocnghia/AutoGenerateReExportFile/tree/master/examples) - -What do you think about this pattern? Please express your idea diff --git a/topics/frontend/remix-versus-nextjs.md b/topics/frontend/remix-versus-nextjs.md deleted file mode 100644 index d2a2905dc..000000000 --- a/topics/frontend/remix-versus-nextjs.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -title: Remix versus Nextjs -description: A brief comparison between Remix and Next.js frameworks -date: 2022-05-27 -authors: - - trankhacvy -tags: - - engineering - - frontend - - radio-talk -redirect: - - /xxMcBA ---- - -A brief comparison between Remix and Next.js frameworks - -## Introduction of Remix - -Remix describes itself as: - -> Remix is an edge native, full-stack JavaScript framework for building modern, fast, and resilient user experiences. It unifies the client and server with web standards so you can think less about code and more about your product. - -Compared to Next.js, which is one of the most React frameworks used for server-side rendering and has been there for a significant time, Remix appears as a new strong competitor. Developers have started to wonder what the difference between these two frameworks is. - -In this post, we will mainly focus on Remix and highlight what makes it unique and where it shines the most. - -## How is Remix different from Next.js? - -### Page rendering strategies - -Despite the same idea of creating websites that are rendered on the server before being sent to the client, Remix and Next.js go with distinct approaches. - -Remix doesn't support Static Site Generation (SSG) but it suggests using HTTP stale-while-revalidate caching directive (SWR) as an alternative. The key here is that static routes are cached using a CDN. These routes are then served to users on each visit and automatically revalidated for the next visitor. - -### Routing - -Both Remix and Next.js follow a file-based routing system. - -In Next.js, each file inside the `/pages` directory will be automatically set as a route. - -```javascript -pages/index.js ==> / -pages/users/index.js ==> /users -pages/users/create.js ==> /users/create -``` - -Remix also does the same but the route directory is `/app/routes`: - -```javascript -app/routes/index.js ==> / -app/routes/users/index.js ==> /users -app/routes/users/create.js ==> /users/create -``` - -Built on top of React Router, Remix shines when it comes to nested routing. It comes with a very powerful route nesting mechanism that can put one or many routes inside another route and those act like children components that can be mounted and unmounted depending on the active URL path. We need to use an Outlet component to render the route hierarchy from the parent routes. - -Next.js, on the other hand, comes with its own router and has support for routes nesting but it’s not so easy to do so compared to Remix. - -### Data loading - -Next.js supports different ways for loading data on the server-side like `getServerSideProps` and `getStaticProps` based on the type of web app. - -Remix has a new concept with two portions: - -- `loader` function - used to load data in a component, and a supplementary -- `useLoaderData hook` - used to get access to the data returned by the `loader` function - -```javascript -import { useLoaderData } from "remix"; - -export const loader = () => { - // fetch data from database or make API calls - return {data} -}; - -export default function App() { - // get access to data - let {data} = useLoaderData(); - - return ( -
-

Use Data in component {data} -

- ); -} -``` - -### Data mutation - -Next.js doesn't do anything for mutations. Basically, we would have to handle everything from creating a form, fetching data, managing state, adding event handlers, to finally dealing with errors, interruptions, and race conditions. - -Data mutations in Remix are built on top of two fundamental web APIs: `
` and HTTP. All we have to do is to use a `` component (works identically to `` with a couple of extra goodies for optimistic UI etc.) and a route function named `action`. When the user submits a form, Remix with call the action for the form and reload all of the data on the page. This ensures that any updates to the data are reflected in the UI. - -```javascript -export const action = async ({ request }) => { - const form = await request.formData(); - const content = form.get("content"); - return redirect("/"); -}; - -export default function App() { - return ( -
- -